From e1fe80cbc07159c2274ae7b96ed2d39c815058ec Mon Sep 17 00:00:00 2001 From: Rohnak Agarwal Date: Tue, 14 Dec 2021 11:27:05 +0530 Subject: [PATCH 001/156] check-transactions stub --- .../processor/bulk/camel/routes/Routes.java | 31 +++++++++++++++ .../processor/bulk/zeebe/ZeebeWorkers.java | 39 ++++++++++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java new file mode 100644 index 00000000..9c15637e --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java @@ -0,0 +1,31 @@ +package org.mifos.processor.bulk.camel.routes; + + +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.util.json.JsonArray; +import org.springframework.stereotype.Component; + +@Component +public class Routes extends RouteBuilder { + + + @Override + public void configure() throws Exception { + routeCheckTransactions(); + } + + private void routeCheckTransactions(){ + String id = "check-transactions"; + from("direct:"+id) + .id(id) + .log("Fetching transaction details") + //set request params + .to("/api/v1/batch/transactions") + .process(exchange -> { + // get response body + // check successful transactions >= x% + // sample y% of successful transactions + // store the sampled request IDs in zeebe workflow variable + }); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java index c940b595..f8ad0bef 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java @@ -5,6 +5,11 @@ import com.fasterxml.jackson.dataformat.csv.CsvSchema; import io.camunda.zeebe.client.ZeebeClient; import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.zeebe.client.api.response.ActivatedJob; +import org.apache.camel.CamelContext; +import org.apache.camel.Exchange; +import org.apache.camel.ProducerTemplate; +import org.apache.camel.support.DefaultExchange; import org.mifos.processor.bulk.file.FileTransferService; import org.mifos.processor.bulk.schema.Transaction; import org.slf4j.Logger; @@ -16,9 +21,10 @@ import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; +import java.util.List; import java.util.Map; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @Component public class ZeebeWorkers { @@ -52,9 +58,19 @@ public class ZeebeWorkers { @Value(value = "${kafka.topic.slcb.name}") private String slcbTopicName; + @Autowired + private CamelContext camelContext; + + @Autowired + private ProducerTemplate producerTemplate; + @PostConstruct public void setupWorkers() { + workerBulkProcessor(); + workerCheckTransactions(); + } + private void workerBulkProcessor(){ zeebeClient.newWorker() .jobType("bulk-processor") .handler((client, job) -> { @@ -85,6 +101,27 @@ else if (current.getPayment_mode().equals("sclb")) .name("bulk-processor") .maxJobsActive(workerMaxJobs) .open(); + } + + private void workerCheckTransactions(){ + String jobType = "check-transactions"; + zeebeClient.newWorker() + .jobType(jobType) + .handler((client, job) -> { + logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); + Map variables = job.getVariablesAsMap(); + String batchId = (String) variables.get(BATCH_ID); + + Exchange exchange = new DefaultExchange(camelContext); + exchange.setProperty("batchId", batchId); + producerTemplate.send("direct:check-transactions", exchange); + + client.newCompleteCommand(job.getKey()) + .send(); + }) + .name(jobType) + .maxJobsActive(workerMaxJobs) + .open(); } } \ No newline at end of file From f9deac51a4cb61597724b19303bde17e037cee34 Mon Sep 17 00:00:00 2001 From: Rohnak Agarwal Date: Tue, 14 Dec 2021 11:37:36 +0530 Subject: [PATCH 002/156] sample-transactions stub --- .../processor/bulk/camel/routes/Routes.java | 18 +++++++++++++-- .../processor/bulk/zeebe/ZeebeWorkers.java | 23 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java index 9c15637e..8d344cb2 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java @@ -12,6 +12,7 @@ public class Routes extends RouteBuilder { @Override public void configure() throws Exception { routeCheckTransactions(); + routeSampleTransactions(); } private void routeCheckTransactions(){ @@ -24,8 +25,21 @@ private void routeCheckTransactions(){ .process(exchange -> { // get response body // check successful transactions >= x% - // sample y% of successful transactions - // store the sampled request IDs in zeebe workflow variable + // set zeebe variable readyForSample = true + }); + } + + private void routeSampleTransactions(){ + String id = "sample-transactions"; + from("direct:" + id) + .id(id) + .log("Fetching transaction details") + //set request params + .to("/api/v1/batch/transactions") + .process(exchange -> { + // get response body + // sample transactions + // store the sampled transaction ids in zeebe variable }); } } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java index f8ad0bef..c9180aa0 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java @@ -68,6 +68,7 @@ public class ZeebeWorkers { public void setupWorkers() { workerBulkProcessor(); workerCheckTransactions(); + workerSampleTransactions(); } private void workerBulkProcessor(){ @@ -124,4 +125,26 @@ private void workerCheckTransactions(){ .maxJobsActive(workerMaxJobs) .open(); } + + private void workerSampleTransactions(){ + String jobType = "sample-transactions"; + zeebeClient.newWorker() + .jobType(jobType) + .handler((client, job) -> { + logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); + + Map variables = job.getVariablesAsMap(); + String batchId = (String) variables.get(BATCH_ID); + + Exchange exchange = new DefaultExchange(camelContext); + exchange.setProperty("batchId", batchId); + producerTemplate.send("direct:sample-transactions", exchange); + + client.newCompleteCommand(job.getKey()) + .send(); + }) + .name(jobType) + .maxJobsActive(workerMaxJobs) + .open(); + } } \ No newline at end of file From 7357815c4835f4b45e8cd6ad1bac2b574437d696 Mon Sep 17 00:00:00 2001 From: Rohnak Agarwal Date: Mon, 27 Dec 2021 18:43:07 +0530 Subject: [PATCH 003/156] stub filled with implementation --- build.gradle | 1 + .../bulk/camel/config/CamelProperties.java | 1 + .../processor/bulk/camel/routes/Routes.java | 85 +++++++++++++++++-- .../processor/bulk/zeebe/ZeebeVariables.java | 2 + .../processor/bulk/zeebe/ZeebeWorkers.java | 6 +- src/main/resources/application.yaml | 4 +- 6 files changed, 88 insertions(+), 11 deletions(-) diff --git a/build.gradle b/build.gradle index 60be2b26..c35b3dc4 100644 --- a/build.gradle +++ b/build.gradle @@ -20,6 +20,7 @@ repositories { } dependencies { + implementation 'org.json:json:20210307' implementation 'org.mifos:ph-ee-connector-common:1.0.0-SNAPSHOT' implementation 'org.apache.camel.springboot:camel-spring-boot-starter:3.4.0' implementation 'org.apache.camel:camel-undertow:3.4.0' diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 310ae182..0f3ffb1e 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -5,5 +5,6 @@ public class CamelProperties { private CamelProperties() {} public static final String AUTH_TYPE = "authType"; + public static final String IS_BATCH_READY = "isBatchReady"; // camel property to check if batch is ready for sampling } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java index 8d344cb2..3c607fad 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java @@ -1,13 +1,30 @@ package org.mifos.processor.bulk.camel.routes; +import io.camunda.zeebe.client.ZeebeClient; import org.apache.camel.builder.RouteBuilder; -import org.apache.camel.util.json.JsonArray; +import org.json.JSONArray; +import org.json.JSONObject; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.IS_BATCH_READY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; + @Component public class Routes extends RouteBuilder { + @Autowired + ZeebeClient zeebeClient; + + @Value("${operations-app-host}") + String operationsAppHost; @Override public void configure() throws Exception { @@ -15,31 +32,83 @@ public void configure() throws Exception { routeSampleTransactions(); } - private void routeCheckTransactions(){ + private void routeCheckTransactions() { String id = "check-transactions"; - from("direct:"+id) + from("direct:" + id) .id(id) .log("Fetching transaction details") //set request params - .to("/api/v1/batch/transactions") + .to(operationsAppHost+ "/api/v1/batch/transactions") .process(exchange -> { // get response body + JSONObject transfers = new JSONObject(exchange.getIn().getBody(String.class)); + + int totalTransactions = transfers.length(); + int successfulTransactions = 0; + for (Iterator it = transfers.keys(); it.hasNext(); ) { + String transactionId = it.next(); + String transactionStatus = transfers.getString(transactionId); + if (transactionStatus.equals("COMPLETED")) { + successfulTransactions++; + } + } + + HashMap newVariables = new HashMap<>(); // check successful transactions >= x% - // set zeebe variable readyForSample = true + if ((successfulTransactions * 100 / totalTransactions) >= 90) { + newVariables.put(IS_SAMPLE_READY, true); + } else { + newVariables.put(IS_SAMPLE_READY, false); + } + + zeebeClient.newSetVariablesCommand(Long.parseLong(exchange.getProperty(BATCH_ID).toString())) + .variables(newVariables) + .send() + .join(); }); } - private void routeSampleTransactions(){ + private void routeSampleTransactions() { String id = "sample-transactions"; from("direct:" + id) .id(id) .log("Fetching transaction details") - //set request params - .to("/api/v1/batch/transactions") + .process(exchange -> { + exchange.getIn().setHeader("batchId", exchange.getProperty(BATCH_ID)); + }) + .toD(operationsAppHost + "/api/v1/batch/transactions") .process(exchange -> { // get response body + + // check if batch is ready for sampling + if (exchange.getProperty(IS_BATCH_READY, String.class).equals("false")) { + return; + } // sample transactions + JSONObject transfers = new JSONObject(exchange.getIn().getBody(String.class)); + final ArrayList successfulTransactionIds = new ArrayList<>(); + final ArrayList sampledTransactionIds = new ArrayList<>(); + for (Iterator it = transfers.keys(); it.hasNext(); ) { + String transactionId = it.next(); + String transactionStatus = transfers.getString(transactionId); + if (transactionStatus.equals("COMPLETED")) { + successfulTransactionIds.add(transactionId); + } + } + Collections.shuffle(successfulTransactionIds); + int sampleSize = (int) (successfulTransactionIds.size() * 0.9); + for (int i = 0; i < sampleSize; i++) { + sampledTransactionIds.add(successfulTransactionIds.get(i)); + } + HashMap newVariables = new HashMap<>(); + newVariables.put(SAMPLED_TX_IDS, sampledTransactionIds.toString()); + // store the sampled transaction ids in zeebe variable + zeebeClient.newSetVariablesCommand(Long.parseLong(exchange.getProperty(BATCH_ID).toString())) + .variables(newVariables) + .send() + .join(); + }); } } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 8802c050..47a18a67 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -22,4 +22,6 @@ private ZeebeVariables() { public static final String INITIATOR_FSPID = "initiatorFspId"; public static final String TRANSACTION_TYPE = "transactionType"; public static final String BATCH_ID = "batchId"; + public static final String IS_SAMPLE_READY = "isSampleReady"; + public static final String SAMPLED_TX_IDS = "sampledTransactionIds"; } \ No newline at end of file diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java index c9180aa0..95840810 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; +import static org.mifos.processor.bulk.camel.config.CamelProperties.IS_BATCH_READY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @Component @@ -115,7 +116,7 @@ private void workerCheckTransactions(){ String batchId = (String) variables.get(BATCH_ID); Exchange exchange = new DefaultExchange(camelContext); - exchange.setProperty("batchId", batchId); + exchange.setProperty(BATCH_ID, batchId); producerTemplate.send("direct:check-transactions", exchange); client.newCompleteCommand(job.getKey()) @@ -137,7 +138,8 @@ private void workerSampleTransactions(){ String batchId = (String) variables.get(BATCH_ID); Exchange exchange = new DefaultExchange(camelContext); - exchange.setProperty("batchId", batchId); + exchange.setProperty(BATCH_ID, batchId); + exchange.setProperty(IS_BATCH_READY, variables.get(IS_SAMPLE_READY)); producerTemplate.send("direct:sample-transactions", exchange); client.newCompleteCommand(job.getKey()) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index bc1cf35d..a4b20967 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -57,4 +57,6 @@ bpmn: gsma-link-based-payment: "gsma_link_transfer" international-remittance-payee: "international_remittance_payee_process-{dfspid}" international-remittance-payer: "international_remittance_payer_process-{dfspid}" - debit-party-process: "debit_party_process-{dfspid}" \ No newline at end of file + debit-party-process: "debit_party_process-{dfspid}" + +operations-app-host: "" \ No newline at end of file From 887feae34b519aa03e3a406344ac95c2e5f9f5e2 Mon Sep 17 00:00:00 2001 From: Rohnak Agarwal Date: Tue, 28 Dec 2021 16:15:54 +0530 Subject: [PATCH 004/156] [FIX] parsing toString -> Gson --- build.gradle | 1 + .../java/org/mifos/processor/bulk/camel/routes/Routes.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index c35b3dc4..87f73f98 100644 --- a/build.gradle +++ b/build.gradle @@ -20,6 +20,7 @@ repositories { } dependencies { + implementation 'com.google.code.gson:gson:2.8.9' implementation 'org.json:json:20210307' implementation 'org.mifos:ph-ee-connector-common:1.0.0-SNAPSHOT' implementation 'org.apache.camel.springboot:camel-spring-boot-starter:3.4.0' diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java index 3c607fad..1d818b0f 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java @@ -1,6 +1,7 @@ package org.mifos.processor.bulk.camel.routes; +import com.google.gson.Gson; import io.camunda.zeebe.client.ZeebeClient; import org.apache.camel.builder.RouteBuilder; import org.json.JSONArray; @@ -101,7 +102,7 @@ private void routeSampleTransactions() { sampledTransactionIds.add(successfulTransactionIds.get(i)); } HashMap newVariables = new HashMap<>(); - newVariables.put(SAMPLED_TX_IDS, sampledTransactionIds.toString()); + newVariables.put(SAMPLED_TX_IDS, new Gson().toJson(sampledTransactionIds)); // store the sampled transaction ids in zeebe variable zeebeClient.newSetVariablesCommand(Long.parseLong(exchange.getProperty(BATCH_ID).toString())) From d3bfc763544f7285aadde23663ff243b54b68513 Mon Sep 17 00:00:00 2001 From: Rohnak Agarwal Date: Wed, 29 Dec 2021 20:32:07 +0530 Subject: [PATCH 005/156] configurable min successful tx ratio --- .../java/org/mifos/processor/bulk/camel/routes/Routes.java | 7 +++++-- src/main/resources/application.yaml | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java index 1d818b0f..7d1f6476 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java @@ -24,9 +24,12 @@ public class Routes extends RouteBuilder { @Autowired ZeebeClient zeebeClient; - @Value("${operations-app-host}") + @Value("${operations-app-config.host}") String operationsAppHost; + @Value("${config.minimum-successful-tx-ratio}") + double minimumSuccessfulTxRatio; + @Override public void configure() throws Exception { routeCheckTransactions(); @@ -56,7 +59,7 @@ private void routeCheckTransactions() { HashMap newVariables = new HashMap<>(); // check successful transactions >= x% - if ((successfulTransactions * 100 / totalTransactions) >= 90) { + if (((double)successfulTransactions / totalTransactions) >= minimumSuccessfulTxRatio) { newVariables.put(IS_SAMPLE_READY, true); } else { newVariables.put(IS_SAMPLE_READY, false); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index a4b20967..e6a4407b 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -59,4 +59,8 @@ bpmn: international-remittance-payer: "international_remittance_payer_process-{dfspid}" debit-party-process: "debit_party_process-{dfspid}" -operations-app-host: "" \ No newline at end of file +operations-app-config: + host: "" + +config: + minimum-successful-tx-ratio: 0.90 \ No newline at end of file From cbb69dbfe93bcd22b35342694cf6e8cb429a3450 Mon Sep 17 00:00:00 2001 From: Thomas Kerin Date: Wed, 19 Jan 2022 10:33:35 +0000 Subject: [PATCH 006/156] Github Actions workflow to build project (#5) * Github Actions workflow to build project * Run workflow on PR --- .github/workflows/test.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..0610bee6 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,10 @@ +name: Test +on: [push, pull_request] +jobs: + Assemble: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Build project + run: ./gradlew clean build From 4ea4d6240cad820e6c0bff83a64b47917c08e8a7 Mon Sep 17 00:00:00 2001 From: Thomas Kerin Date: Mon, 31 Jan 2022 04:44:09 +0000 Subject: [PATCH 007/156] Fix Dockerfile: should copy correct filepath (changed after adopting gradle) (#7) * Fix Dockerfile: should copy correct filepath (changed after adopting gradle) * Github Actions: also build Docker image in workflow --- .github/workflows/test.yml | 2 ++ Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0610bee6..f71e4d7d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,3 +8,5 @@ jobs: uses: actions/checkout@v2 - name: Build project run: ./gradlew clean build + - name: Build Docker image + run: docker build . -t paymenthubee.mifos.io/phee/bulk-processor diff --git a/Dockerfile b/Dockerfile index 7424e18f..67ec2e5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM openjdk:13 EXPOSE 5000 -COPY target/*.jar . -CMD java -jar *.jar \ No newline at end of file +COPY build/libs/*.jar . +CMD java -jar *.jar From 49f5f4dc9e3dc7f51f35a006721c7140120ad5ee Mon Sep 17 00:00:00 2001 From: Thomas Kerin Date: Mon, 31 Jan 2022 04:45:02 +0000 Subject: [PATCH 008/156] build.gradle - needs org.springframework.boot plugin to start application (#6) --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index 60be2b26..63437233 100644 --- a/build.gradle +++ b/build.gradle @@ -6,6 +6,7 @@ plugins { id 'java' id 'maven-publish' id 'eclipse' + id 'org.springframework.boot' version '2.1.9.RELEASE' } repositories { From e3709090d82a1f969e7d822d9b8c5a9658d6b94d Mon Sep 17 00:00:00 2001 From: Alikhan562 Date: Thu, 17 Feb 2022 14:32:46 +0530 Subject: [PATCH 009/156] removed the mavenlocal and introduce Jfrog url --- build.gradle | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 63437233..78cde056 100644 --- a/build.gradle +++ b/build.gradle @@ -10,9 +10,13 @@ plugins { } repositories { - mavenLocal() + // mavenLocal() maven { url = uri('https://repo.maven.apache.org/maven2') + + } + maven { + url "https://fynarfin.jfrog.io/artifactory/fyn-libs-release-local" } maven { From 2dbc21c75c6558906cfd1d4adc2a06a83ac26d9c Mon Sep 17 00:00:00 2001 From: Rohnak Agarwal Date: Mon, 27 Dec 2021 18:43:07 +0530 Subject: [PATCH 010/156] stub filled with implementation --- build.gradle | 1 + .../bulk/camel/config/CamelProperties.java | 1 + .../processor/bulk/camel/routes/Routes.java | 85 +++++++++++++++++-- .../processor/bulk/zeebe/ZeebeVariables.java | 2 + .../processor/bulk/zeebe/ZeebeWorkers.java | 6 +- src/main/resources/application.yaml | 4 +- 6 files changed, 88 insertions(+), 11 deletions(-) diff --git a/build.gradle b/build.gradle index 78cde056..c452b6b1 100644 --- a/build.gradle +++ b/build.gradle @@ -25,6 +25,7 @@ repositories { } dependencies { + implementation 'org.json:json:20210307' implementation 'org.mifos:ph-ee-connector-common:1.0.0-SNAPSHOT' implementation 'org.apache.camel.springboot:camel-spring-boot-starter:3.4.0' implementation 'org.apache.camel:camel-undertow:3.4.0' diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 310ae182..0f3ffb1e 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -5,5 +5,6 @@ public class CamelProperties { private CamelProperties() {} public static final String AUTH_TYPE = "authType"; + public static final String IS_BATCH_READY = "isBatchReady"; // camel property to check if batch is ready for sampling } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java index 8d344cb2..3c607fad 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java @@ -1,13 +1,30 @@ package org.mifos.processor.bulk.camel.routes; +import io.camunda.zeebe.client.ZeebeClient; import org.apache.camel.builder.RouteBuilder; -import org.apache.camel.util.json.JsonArray; +import org.json.JSONArray; +import org.json.JSONObject; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.IS_BATCH_READY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; + @Component public class Routes extends RouteBuilder { + @Autowired + ZeebeClient zeebeClient; + + @Value("${operations-app-host}") + String operationsAppHost; @Override public void configure() throws Exception { @@ -15,31 +32,83 @@ public void configure() throws Exception { routeSampleTransactions(); } - private void routeCheckTransactions(){ + private void routeCheckTransactions() { String id = "check-transactions"; - from("direct:"+id) + from("direct:" + id) .id(id) .log("Fetching transaction details") //set request params - .to("/api/v1/batch/transactions") + .to(operationsAppHost+ "/api/v1/batch/transactions") .process(exchange -> { // get response body + JSONObject transfers = new JSONObject(exchange.getIn().getBody(String.class)); + + int totalTransactions = transfers.length(); + int successfulTransactions = 0; + for (Iterator it = transfers.keys(); it.hasNext(); ) { + String transactionId = it.next(); + String transactionStatus = transfers.getString(transactionId); + if (transactionStatus.equals("COMPLETED")) { + successfulTransactions++; + } + } + + HashMap newVariables = new HashMap<>(); // check successful transactions >= x% - // set zeebe variable readyForSample = true + if ((successfulTransactions * 100 / totalTransactions) >= 90) { + newVariables.put(IS_SAMPLE_READY, true); + } else { + newVariables.put(IS_SAMPLE_READY, false); + } + + zeebeClient.newSetVariablesCommand(Long.parseLong(exchange.getProperty(BATCH_ID).toString())) + .variables(newVariables) + .send() + .join(); }); } - private void routeSampleTransactions(){ + private void routeSampleTransactions() { String id = "sample-transactions"; from("direct:" + id) .id(id) .log("Fetching transaction details") - //set request params - .to("/api/v1/batch/transactions") + .process(exchange -> { + exchange.getIn().setHeader("batchId", exchange.getProperty(BATCH_ID)); + }) + .toD(operationsAppHost + "/api/v1/batch/transactions") .process(exchange -> { // get response body + + // check if batch is ready for sampling + if (exchange.getProperty(IS_BATCH_READY, String.class).equals("false")) { + return; + } // sample transactions + JSONObject transfers = new JSONObject(exchange.getIn().getBody(String.class)); + final ArrayList successfulTransactionIds = new ArrayList<>(); + final ArrayList sampledTransactionIds = new ArrayList<>(); + for (Iterator it = transfers.keys(); it.hasNext(); ) { + String transactionId = it.next(); + String transactionStatus = transfers.getString(transactionId); + if (transactionStatus.equals("COMPLETED")) { + successfulTransactionIds.add(transactionId); + } + } + Collections.shuffle(successfulTransactionIds); + int sampleSize = (int) (successfulTransactionIds.size() * 0.9); + for (int i = 0; i < sampleSize; i++) { + sampledTransactionIds.add(successfulTransactionIds.get(i)); + } + HashMap newVariables = new HashMap<>(); + newVariables.put(SAMPLED_TX_IDS, sampledTransactionIds.toString()); + // store the sampled transaction ids in zeebe variable + zeebeClient.newSetVariablesCommand(Long.parseLong(exchange.getProperty(BATCH_ID).toString())) + .variables(newVariables) + .send() + .join(); + }); } } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 8802c050..47a18a67 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -22,4 +22,6 @@ private ZeebeVariables() { public static final String INITIATOR_FSPID = "initiatorFspId"; public static final String TRANSACTION_TYPE = "transactionType"; public static final String BATCH_ID = "batchId"; + public static final String IS_SAMPLE_READY = "isSampleReady"; + public static final String SAMPLED_TX_IDS = "sampledTransactionIds"; } \ No newline at end of file diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java index c9180aa0..95840810 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; +import static org.mifos.processor.bulk.camel.config.CamelProperties.IS_BATCH_READY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @Component @@ -115,7 +116,7 @@ private void workerCheckTransactions(){ String batchId = (String) variables.get(BATCH_ID); Exchange exchange = new DefaultExchange(camelContext); - exchange.setProperty("batchId", batchId); + exchange.setProperty(BATCH_ID, batchId); producerTemplate.send("direct:check-transactions", exchange); client.newCompleteCommand(job.getKey()) @@ -137,7 +138,8 @@ private void workerSampleTransactions(){ String batchId = (String) variables.get(BATCH_ID); Exchange exchange = new DefaultExchange(camelContext); - exchange.setProperty("batchId", batchId); + exchange.setProperty(BATCH_ID, batchId); + exchange.setProperty(IS_BATCH_READY, variables.get(IS_SAMPLE_READY)); producerTemplate.send("direct:sample-transactions", exchange); client.newCompleteCommand(job.getKey()) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index bc1cf35d..a4b20967 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -57,4 +57,6 @@ bpmn: gsma-link-based-payment: "gsma_link_transfer" international-remittance-payee: "international_remittance_payee_process-{dfspid}" international-remittance-payer: "international_remittance_payer_process-{dfspid}" - debit-party-process: "debit_party_process-{dfspid}" \ No newline at end of file + debit-party-process: "debit_party_process-{dfspid}" + +operations-app-host: "" \ No newline at end of file From a21c24d408e124a55deaeb0c494c2fecaa321d9c Mon Sep 17 00:00:00 2001 From: Rohnak Agarwal Date: Tue, 28 Dec 2021 16:15:54 +0530 Subject: [PATCH 011/156] [FIX] parsing toString -> Gson --- build.gradle | 1 + .../java/org/mifos/processor/bulk/camel/routes/Routes.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index c452b6b1..f7a584de 100644 --- a/build.gradle +++ b/build.gradle @@ -25,6 +25,7 @@ repositories { } dependencies { + implementation 'com.google.code.gson:gson:2.8.9' implementation 'org.json:json:20210307' implementation 'org.mifos:ph-ee-connector-common:1.0.0-SNAPSHOT' implementation 'org.apache.camel.springboot:camel-spring-boot-starter:3.4.0' diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java index 3c607fad..1d818b0f 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java @@ -1,6 +1,7 @@ package org.mifos.processor.bulk.camel.routes; +import com.google.gson.Gson; import io.camunda.zeebe.client.ZeebeClient; import org.apache.camel.builder.RouteBuilder; import org.json.JSONArray; @@ -101,7 +102,7 @@ private void routeSampleTransactions() { sampledTransactionIds.add(successfulTransactionIds.get(i)); } HashMap newVariables = new HashMap<>(); - newVariables.put(SAMPLED_TX_IDS, sampledTransactionIds.toString()); + newVariables.put(SAMPLED_TX_IDS, new Gson().toJson(sampledTransactionIds)); // store the sampled transaction ids in zeebe variable zeebeClient.newSetVariablesCommand(Long.parseLong(exchange.getProperty(BATCH_ID).toString())) From a454f2d36c69f79adb25ab0e410c9f26fdcffd34 Mon Sep 17 00:00:00 2001 From: Rohnak Agarwal Date: Wed, 29 Dec 2021 20:32:07 +0530 Subject: [PATCH 012/156] configurable min successful tx ratio --- .../java/org/mifos/processor/bulk/camel/routes/Routes.java | 7 +++++-- src/main/resources/application.yaml | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java index 1d818b0f..7d1f6476 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java @@ -24,9 +24,12 @@ public class Routes extends RouteBuilder { @Autowired ZeebeClient zeebeClient; - @Value("${operations-app-host}") + @Value("${operations-app-config.host}") String operationsAppHost; + @Value("${config.minimum-successful-tx-ratio}") + double minimumSuccessfulTxRatio; + @Override public void configure() throws Exception { routeCheckTransactions(); @@ -56,7 +59,7 @@ private void routeCheckTransactions() { HashMap newVariables = new HashMap<>(); // check successful transactions >= x% - if ((successfulTransactions * 100 / totalTransactions) >= 90) { + if (((double)successfulTransactions / totalTransactions) >= minimumSuccessfulTxRatio) { newVariables.put(IS_SAMPLE_READY, true); } else { newVariables.put(IS_SAMPLE_READY, false); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index a4b20967..e6a4407b 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -59,4 +59,8 @@ bpmn: international-remittance-payer: "international_remittance_payer_process-{dfspid}" debit-party-process: "debit_party_process-{dfspid}" -operations-app-host: "" \ No newline at end of file +operations-app-config: + host: "" + +config: + minimum-successful-tx-ratio: 0.90 \ No newline at end of file From b5839f9781c0713cd6d05f727086f8d82bc1323d Mon Sep 17 00:00:00 2001 From: Danish Jamal Date: Thu, 28 Apr 2022 18:08:00 +0530 Subject: [PATCH 013/156] Minor bug fixed --- .DS_Store | Bin 6148 -> 0 bytes .mvn/wrapper/MavenWrapperDownloader.java | 118 ------------------ .mvn/wrapper/maven-wrapper.jar | Bin 50710 -> 0 bytes .mvn/wrapper/maven-wrapper.properties | 2 - build.gradle | 2 +- src/.DS_Store | Bin 6148 -> 0 bytes src/main/.DS_Store | Bin 6148 -> 0 bytes src/main/java/.DS_Store | Bin 6148 -> 0 bytes src/main/java/org/.DS_Store | Bin 6148 -> 0 bytes src/main/java/org/mifos/.DS_Store | Bin 6148 -> 0 bytes .../processor/bulk/OperationsAppConfig.java | 23 ++++ .../bulk/camel/routes/BaseRouteBuilder.java | 57 +++++++++ .../processor/bulk/camel/routes/Routes.java | 12 +- src/main/resources/application.yaml | 10 +- .../BulkProcessorApplicationTests.java | 2 +- 15 files changed, 91 insertions(+), 135 deletions(-) delete mode 100644 .DS_Store delete mode 100644 .mvn/wrapper/MavenWrapperDownloader.java delete mode 100644 .mvn/wrapper/maven-wrapper.jar delete mode 100644 .mvn/wrapper/maven-wrapper.properties delete mode 100644 src/.DS_Store delete mode 100644 src/main/.DS_Store delete mode 100644 src/main/java/.DS_Store delete mode 100644 src/main/java/org/.DS_Store delete mode 100644 src/main/java/org/mifos/.DS_Store create mode 100644 src/main/java/org/mifos/processor/bulk/OperationsAppConfig.java create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java rename src/test/java/org/mifos/{pheeprocessorbulk => processor}/BulkProcessorApplicationTests.java (88%) diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 3fa89eb1a150b87c1d5bb719b7a2801972264bc8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK&1%~~5T3Q&WbIPoLm`0@&?Ps=Sk4d4At2oJkXu2eIkX{>ERzTnt+C`Nt>dBx zX`i7l(tDnzr*>v{9k7#}0-=N%nEhsVzge-L>|G52h~7AI0h$0H(FrScZ0<3dC%x%OXedLweigEsUrj^_DdeSyCW zLr}Q6c37%=IeJV&4V}{jC67w4UqEXC{80)^wcTv0g(nO`S{~3G6Vq^ta0aoC#3fOZ+zI^OLoX-ld0?$qXIv*T# zLeFApP#+!G=n??2hQr!0)}Q`}4h4Ao|OWkcHkfMa7*@X zy|X#GYaR4cbTZauOywyU~q+qUhh+uxis_~*a##hm*_WW?9E7Pb7N%LRFiwbEGCJ0XP=%-6oeT$XZcYgtzC2~q zk(K08IQL8oTl}>>+hE5YRgXTB@fZ4TH9>7=79e`%%tw*SQUa9~$xKD5rS!;ZG@ocK zQdcH}JX?W|0_Afv?y`-NgLum62B&WSD$-w;O6G0Sm;SMX65z)l%m1e-g8Q$QTI;(Q z+x$xth4KFvH@Bs6(zn!iF#nenk^Y^ce;XIItAoCsow38eq?Y-Auh!1in#Rt-_D>H^ z=EjbclGGGa6VnaMGmMLj`x3NcwA43Jb(0gzl;RUIRAUDcR1~99l2SAPkVhoRMMtN} zXvC<tOmX83grD8GSo_Lo?%lNfhD#EBgPo z*nf@ppMC#B!T)Ae0RG$mlJWmGl7CkuU~B8-==5i;rS;8i6rJ=PoQxf446XDX9g|c> zU64ePyMlsI^V5Jq5A+BPe#e73+kpc_r1tv#B)~EZ;7^67F0*QiYfrk0uVW;Qb=NsG zN>gsuCwvb?s-KQIppEaeXtEMdc9dy6Dfduz-tMTms+i01{eD9JE&h?Kht*$eOl#&L zJdM_-vXs(V#$Ed;5wyNWJdPNh+Z$+;$|%qR(t`4W@kDhd*{(7-33BOS6L$UPDeE_53j${QfKN-0v-HG z(QfyvFNbwPK%^!eIo4ac1;b>c0vyf9}Xby@YY!lkz-UvNp zwj#Gg|4B~?n?G^{;(W;|{SNoJbHTMpQJ*Wq5b{l9c8(%?Kd^1?H1om1de0Da9M;Q=n zUfn{f87iVb^>Exl*nZ0hs(Yt>&V9$Pg`zX`AI%`+0SWQ4Zc(8lUDcTluS z5a_KerZWe}a-MF9#Cd^fi!y3%@RFmg&~YnYZ6<=L`UJ0v={zr)>$A;x#MCHZy1st7 ztT+N07NR+vOwSV2pvWuN1%lO!K#Pj0Fr>Q~R40{bwdL%u9i`DSM4RdtEH#cW)6}+I-eE< z&tZs+(Ogu(H_;$a$!7w`MH0r%h&@KM+<>gJL@O~2K2?VrSYUBbhCn#yy?P)uF3qWU z0o09mIik+kvzV6w>vEZy@&Mr)SgxPzUiDA&%07m17udz9usD82afQEps3$pe!7fUf z0eiidkJ)m3qhOjVHC_M(RYCBO%CZKZXFb8}s0-+}@CIn&EF(rRWUX2g^yZCvl0bI} zbP;1S)iXnRC&}5-Tl(hASKqdSnO?ASGJ*MIhOXIblmEudj(M|W!+I3eDc}7t`^mtg z)PKlaXe(OH+q-)qcQ8a@!llRrpGI8DsjhoKvw9T;TEH&?s=LH0w$EzI>%u;oD@x83 zJL7+ncjI9nn!TlS_KYu5vn%f*@qa5F;| zEFxY&B?g=IVlaF3XNm_03PA)=3|{n-UCgJoTr;|;1AU9|kPE_if8!Zvb}0q$5okF$ zHaJdmO&gg!9oN|M{!qGE=tb|3pVQ8PbL$}e;NgXz<6ZEggI}wO@aBP**2Wo=yN#ZC z4G$m^yaM9g=|&!^ft8jOLuzc3Psca*;7`;gnHm}tS0%f4{|VGEwu45KptfNmwxlE~ z^=r30gi@?cOm8kAz!EylA4G~7kbEiRlRIzwrb~{_2(x^$-?|#e6Bi_**(vyr_~9Of z!n>Gqf+Qwiu!xhi9f53=PM3`3tNF}pCOiPU|H4;pzjcsqbwg*{{kyrTxk<;mx~(;; z1NMrpaQ`57yn34>Jo3b|HROE(UNcQash!0p2-!Cz;{IRv#Vp5!3o$P8!%SgV~k&Hnqhp`5eLjTcy93cK!3Hm-$`@yGnaE=?;*2uSpiZTs_dDd51U%i z{|Zd9ou-;laGS_x=O}a+ zB||za<795A?_~Q=r=coQ+ZK@@ zId~hWQL<%)fI_WDIX#=(WNl!Dm$a&ROfLTd&B$vatq!M-2Jcs;N2vps$b6P1(N}=oI3<3luMTmC|0*{ zm1w8bt7vgX($!0@V0A}XIK)w!AzUn7vH=pZEp0RU0p?}ch2XC-7r#LK&vyc2=-#Q2 z^L%8)JbbcZ%g0Du;|8=q8B>X=mIQirpE=&Ox{TiuNDnOPd-FLI^KfEF729!!0x#Es z@>3ursjFSpu%C-8WL^Zw!7a0O-#cnf`HjI+AjVCFitK}GXO`ME&on|^=~Zc}^LBp9 zj=-vlN;Uc;IDjtK38l7}5xxQF&sRtfn4^TNtnzXv4M{r&ek*(eNbIu!u$>Ed%` z5x7+&)2P&4>0J`N&ZP8$vcR+@FS0126s6+Jx_{{`3ZrIMwaJo6jdrRwE$>IU_JTZ} z(||hyyQ)4Z1@wSlT94(-QKqkAatMmkT7pCycEB1U8KQbFX&?%|4$yyxCtm3=W`$4fiG0WU3yI@c zx{wfmkZAYE_5M%4{J-ygbpH|(|GD$2f$3o_Vti#&zfSGZMQ5_f3xt6~+{RX=$H8at z?GFG1Tmp}}lmm-R->ve*Iv+XJ@58p|1_jRvfEgz$XozU8#iJS})UM6VNI!3RUU!{5 zXB(+Eqd-E;cHQ>)`h0(HO_zLmzR3Tu-UGp;08YntWwMY-9i^w_u#wR?JxR2bky5j9 z3Sl-dQQU$xrO0xa&>vsiK`QN<$Yd%YXXM7*WOhnRdSFt5$aJux8QceC?lA0_if|s> ze{ad*opH_kb%M&~(~&UcX0nFGq^MqjxW?HJIP462v9XG>j(5Gat_)#SiNfahq2Mz2 zU`4uV8m$S~o9(W>mu*=h%Gs(Wz+%>h;R9Sg)jZ$q8vT1HxX3iQnh6&2rJ1u|j>^Qf`A76K%_ubL`Zu?h4`b=IyL>1!=*%!_K)=XC z6d}4R5L+sI50Q4P3upXQ3Z!~1ZXLlh!^UNcK6#QpYt-YC=^H=EPg3)z*wXo*024Q4b2sBCG4I# zlTFFY=kQ>xvR+LsuDUAk)q%5pEcqr(O_|^spjhtpb1#aC& zghXzGkGDC_XDa%t(X`E+kvKQ4zrQ*uuQoj>7@@ykWvF332)RO?%AA&Fsn&MNzmFa$ zWk&&^=NNjxLjrli_8ESU)}U|N{%j&TQmvY~lk!~Jh}*=^INA~&QB9em!in_X%Rl1&Kd~Z(u z9mra#<@vZQlOY+JYUwCrgoea4C8^(xv4ceCXcejq84TQ#sF~IU2V}LKc~Xlr_P=ry zl&Hh0exdCbVd^NPCqNNlxM3vA13EI8XvZ1H9#bT7y*U8Y{H8nwGpOR!e!!}*g;mJ#}T{ekSb}5zIPmye*If(}}_=PcuAW#yidAa^9-`<8Gr0 z)Fz=NiZ{)HAvw{Pl5uu)?)&i&Us$Cx4gE}cIJ}B4Xz~-q7)R_%owbP!z_V2=Aq%Rj z{V;7#kV1dNT9-6R+H}}(ED*_!F=~uz>&nR3gb^Ce%+0s#u|vWl<~JD3MvS0T9thdF zioIG3c#Sdsv;LdtRv3ml7%o$6LTVL>(H`^@TNg`2KPIk*8-IB}X!MT0`hN9Ddf7yN z?J=GxPL!uJ7lqwowsl?iRrh@#5C$%E&h~Z>XQcvFC*5%0RN-Opq|=IwX(dq(*sjs+ zqy99+v~m|6T#zR*e1AVxZ8djd5>eIeCi(b8sUk)OGjAsKSOg^-ugwl2WSL@d#?mdl zib0v*{u-?cq}dDGyZ%$XRY=UkQwt2oGu`zQneZh$=^! zj;!pCBWQNtvAcwcWIBM2y9!*W|8LmQy$H~5BEx)78J`4Z0(FJO2P^!YyQU{*Al+fs z){!4JvT1iLrJ8aU3k0t|P}{RN)_^v%$$r;+p0DY7N8CXzmS*HB*=?qaaF9D@#_$SN zSz{moAK<*RH->%r7xX~9gVW$l7?b|_SYI)gcjf0VAUJ%FcQP(TpBs; zg$25D!Ry_`8xpS_OJdeo$qh#7U+cepZ??TII7_%AXsT$B z=e)Bx#v%J0j``00Zk5hsvv6%T^*xGNx%KN-=pocSoqE5_R)OK%-Pbu^1MNzfds)mL zxz^F4lDKV9D&lEY;I+A)ui{TznB*CE$=9(wgE{m}`^<--OzV-5V4X2w9j(_!+jpTr zJvD*y6;39&T+==$F&tsRKM_lqa1HC}aGL0o`%c9mO=fts?36@8MGm7Vi{Y z^<7m$(EtdSr#22<(rm_(l_(`j!*Pu~Y>>xc>I9M#DJYDJNHO&4=HM%YLIp?;iR&$m z#_$ZWYLfGLt5FJZhr3jpYb`*%9S!zCG6ivNHYzNHcI%khtgHBliM^Ou}ZVD7ehU9 zS+W@AV=?Ro!=%AJ>Kcy9aU3%VX3|XM_K0A+ZaknKDyIS3S-Hw1C7&BSW5)sqj5Ye_ z4OSW7Yu-;bCyYKHFUk}<*<(@TH?YZPHr~~Iy%9@GR2Yd}J2!N9K&CN7Eq{Ka!jdu; zQNB*Y;i(7)OxZK%IHGt#Rt?z`I|A{q_BmoF!f^G}XVeTbe1Wnzh%1g>j}>DqFf;Rp zz7>xIs12@Ke0gr+4-!pmFP84vCIaTjqFNg{V`5}Rdt~xE^I;Bxp4)|cs8=f)1YwHz zqI`G~s2~qqDV+h02b`PQpUE#^^Aq8l%y2|ByQeXSADg5*qMprEAE3WFg0Q39`O+i1 z!J@iV!`Y~C$wJ!5Z+j5$i<1`+@)tBG$JL=!*uk=2k;T<@{|s1$YL079FvK%mPhyHV zP8^KGZnp`(hVMZ;s=n~3r2y;LTwcJwoBW-(ndU-$03{RD zh+Qn$ja_Z^OuMf3Ub|JTY74s&Am*(n{J3~@#OJNYuEVVJd9*H%)oFoRBkySGm`hx! zT3tG|+aAkXcx-2Apy)h^BkOyFTWQVeZ%e2@;*0DtlG9I3Et=PKaPt&K zw?WI7S;P)TWED7aSH$3hL@Qde?H#tzo^<(o_sv_2ci<7M?F$|oCFWc?7@KBj-;N$P zB;q!8@bW-WJY9do&y|6~mEruZAVe$!?{)N9rZZxD-|oltkhW9~nR8bLBGXw<632!l z*TYQn^NnUy%Ds}$f^=yQ+BM-a5X4^GHF=%PDrRfm_uqC zh{sKwIu|O0&jWb27;wzg4w5uA@TO_j(1X?8E>5Zfma|Ly7Bklq|s z9)H`zoAGY3n-+&JPrT!>u^qg9Evx4y@GI4$n-Uk_5wttU1_t?6><>}cZ-U+&+~JE) zPlDbO_j;MoxdLzMd~Ew|1o^a5q_1R*JZ=#XXMzg?6Zy!^hop}qoLQlJ{(%!KYt`MK z8umEN@Z4w!2=q_oe=;QttPCQy3Nm4F@x>@v4sz_jo{4m*0r%J(w1cSo;D_hQtJs7W z><$QrmG^+<$4{d2bgGo&3-FV}avg9zI|Rr(k{wTyl3!M1q+a zD9W{pCd%il*j&Ft z5H$nENf>>k$;SONGW`qo6`&qKs*T z2^RS)pXk9b@(_Fw1bkb)-oqK|v}r$L!W&aXA>IpcdNZ_vWE#XO8X`#Yp1+?RshVcd zknG%rPd*4ECEI0wD#@d+3NbHKxl}n^Sgkx==Iu%}HvNliOqVBqG?P2va zQ;kRJ$J6j;+wP9cS za#m;#GUT!qAV%+rdWolk+)6kkz4@Yh5LXP+LSvo9_T+MmiaP-eq6_k;)i6_@WSJ zlT@wK$zqHu<83U2V*yJ|XJU4farT#pAA&@qu)(PO^8PxEmPD4;Txpio+2)#!9 z>&=i7*#tc0`?!==vk>s7V+PL#S1;PwSY?NIXN2=Gu89x(cToFm))7L;< z+bhAbVD*bD=}iU`+PU+SBobTQ%S!=VL!>q$rfWsaaV}Smz>lO9JXT#`CcH_mRCSf4%YQAw`$^yY z3Y*^Nzk_g$xn7a_NO(2Eb*I=^;4f!Ra#Oo~LLjlcjke*k*o$~U#0ZXOQ5@HQ&T46l z7504MUgZkz2gNP1QFN8Y?nSEnEai^Rgyvl}xZfMUV6QrJcXp;jKGqB=D*tj{8(_pV zqyB*DK$2lgYGejmJUW)*s_Cv65sFf&pb(Yz8oWgDtQ0~k^0-wdF|tj}MOXaN@ydF8 zNr={U?=;&Z?wr^VC+`)S2xl}QFagy;$mG=TUs7Vi2wws5zEke4hTa2)>O0U?$WYsZ z<8bN2bB_N4AWd%+kncgknZ&}bM~eDtj#C5uRkp21hWW5gxWvc6b*4+dn<{c?w9Rmf zIVZKsPl{W2vQAlYO3yh}-{Os=YBnL8?uN5(RqfQ=-1cOiUnJu>KcLA*tQK3FU`_bM zM^T28w;nAj5EdAXFi&Kk1Nnl2)D!M{@+D-}bIEe+Lc4{s;YJc-{F#``iS2uk;2!Zp zF9#myUmO!wCeJIoi^A+T^e~20c+c2C}XltaR!|U-HfDA=^xF97ev}$l6#oY z&-&T{egB)&aV$3_aVA51XGiU07$s9vubh_kQG?F$FycvS6|IO!6q zq^>9|3U^*!X_C~SxX&pqUkUjz%!j=VlXDo$!2VLH!rKj@61mDpSr~7B2yy{>X~_nc zRI+7g2V&k zd**H++P9dg!-AOs3;GM`(g<+GRV$+&DdMVpUxY9I1@uK28$az=6oaa+PutlO9?6#? zf-OsgT>^@8KK>ggkUQRPPgC7zjKFR5spqQb3ojCHzj^(UH~v+!y*`Smv)VpVoPwa6 zWG18WJaPKMi*F6Zdk*kU^`i~NNTfn3BkJniC`yN98L-Awd)Z&mY? zprBW$!qL-OL7h@O#kvYnLsfff@kDIegt~?{-*5A7JrA;#TmTe?jICJqhub-G@e??D zqiV#g{)M!kW1-4SDel7TO{;@*h2=_76g3NUD@|c*WO#>MfYq6_YVUP+&8e4|%4T`w zXzhmVNziAHazWO2qXcaOu@R1MrPP{t)`N)}-1&~mq=ZH=w=;-E$IOk=y$dOls{6sRR`I5>|X zpq~XYW4sd;J^6OwOf**J>a7u$S>WTFPRkjY;BfVgQst)u4aMLR1|6%)CB^18XCz+r ztkYQ}G43j~Q&1em(_EkMv0|WEiKu;z2zhb(L%$F&xWwzOmk;VLBYAZ8lOCziNoPw1 zv2BOyXA`A8z^WH!nXhKXM`t0;6D*-uGds3TYGrm8SPnJJOQ^fJU#}@aIy@MYWz**H zvkp?7I5PE{$$|~{-ZaFxr6ZolP^nL##mHOErB^AqJqn^hFA=)HWj!m3WDaHW$C)i^ z9@6G$SzB=>jbe>4kqr#sF7#K}W*Cg-5y6kun3u&0L7BpXF9=#7IN8FOjWrWwUBZiU zT_se3ih-GBKx+Uw0N|CwP3D@-C=5(9T#BH@M`F2!Goiqx+Js5xC92|Sy0%WWWp={$(am!#l~f^W_oz78HX<0X#7 zp)p1u~M*o9W@O8P{0Qkg@Wa# z2{Heb&oX^CQSZWSFBXKOfE|tsAm#^U-WkDnU;IowZ`Ok4!mwHwH=s|AqZ^YD4!5!@ zPxJj+Bd-q6w_YG`z_+r;S86zwXb+EO&qogOq8h-Ect5(M2+>(O7n7)^dP*ws_3U6v zVsh)sk^@*c>)3EML|0<-YROho{lz@Nd4;R9gL{9|64xVL`n!m$-Jjrx?-Bacp!=^5 z1^T^eB{_)Y<9)y{-4Rz@9_>;_7h;5D+@QcbF4Wv7hu)s0&==&6u)33 zHRj+&Woq-vDvjwJCYES@$C4{$?f$Ibi4G()UeN11rgjF+^;YE^5nYprYoJNoudNj= zm1pXSeG64dcWHObUetodRn1Fw|1nI$D9z}dVEYT0lQnsf_E1x2vBLql7NrHH!n&Sq z6lc*mvU=WS6=v9Lrl}&zRiu_6u;6g%_DU{9b+R z#YHqX7`m9eydf?KlKu6Sb%j$%_jmydig`B*TN`cZL-g!R)iE?+Q5oOqBFKhx z%MW>BC^(F_JuG(ayE(MT{S3eI{cKiwOtPwLc0XO*{*|(JOx;uQOfq@lp_^cZo=FZj z4#}@e@dJ>Bn%2`2_WPeSN7si^{U#H=7N4o%Dq3NdGybrZgEU$oSm$hC)uNDC_M9xc zGzwh5Sg?mpBIE8lT2XsqTt3j3?We8}3bzLBTQd639vyg^$0#1epq8snlDJP2(BF)K zSx30RM+{f+b$g{9usIL8H!hCO117Xgv}ttPJm9wVRjPk;ePH@zxv%j9k5`TzdXLeT zFgFX`V7cYIcBls5WN0Pf6SMBN+;CrQ(|EsFd*xtwr#$R{Z9FP`OWtyNsq#mCgZ7+P z^Yn$haBJ)r96{ZJd8vlMl?IBxrgh=fdq_NF!1{jARCVz>jNdC)H^wfy?R94#MPdUjcYX>#wEx+LB#P-#4S-%YH>t-j+w zOFTI8gX$ard6fAh&g=u&56%3^-6E2tpk*wx3HSCQ+t7+*iOs zPk5ysqE}i*cQocFvA68xHfL|iX(C4h*67@3|5Qwle(8wT&!&{8*{f%0(5gH+m>$tq zp;AqrP7?XTEooYG1Dzfxc>W%*CyL16q|fQ0_jp%%Bk^k!i#Nbi(N9&T>#M{gez_Ws zYK=l}adalV(nH}I_!hNeb;tQFk3BHX7N}}R8%pek^E`X}%ou=cx8InPU1EE0|Hen- zyw8MoJqB5=)Z%JXlrdTXAE)eqLAdVE-=>wGHrkRet}>3Yu^lt$Kzu%$3#(ioY}@Gu zjk3BZuQH&~7H+C*uX^4}F*|P89JX;Hg2U!pt>rDi(n(Qe-c}tzb0#6_ItoR0->LSt zR~UT<-|@TO%O`M+_e_J4wx7^)5_%%u+J=yF_S#2Xd?C;Ss3N7KY^#-vx+|;bJX&8r zD?|MetfhdC;^2WG`7MCgs>TKKN=^=!x&Q~BzmQio_^l~LboTNT=I zC5pme^P@ER``p$2md9>4!K#vV-Fc1an7pl>_|&>aqP}+zqR?+~Z;f2^`a+-!Te%V? z;H2SbF>jP^GE(R1@%C==XQ@J=G9lKX+Z<@5}PO(EYkJh=GCv#)Nj{DkWJM2}F&oAZ6xu8&g7pn1ps2U5srwQ7CAK zN&*~@t{`31lUf`O;2w^)M3B@o)_mbRu{-`PrfNpF!R^q>yTR&ETS7^-b2*{-tZAZz zw@q5x9B5V8Qd7dZ!Ai$9hk%Q!wqbE1F1c96&zwBBaRW}(^axoPpN^4Aw}&a5dMe+*Gomky_l^54*rzXro$ z>LL)U5Ry>~FJi=*{JDc)_**c)-&faPz`6v`YU3HQa}pLtb5K)u%K+BOqXP0)rj5Au$zB zW1?vr?mDv7Fsxtsr+S6ucp2l#(4dnr9sD*v+@*>g#M4b|U?~s93>Pg{{a5|rm2xfI z`>E}?9S@|IoUX{Q1zjm5YJT|3S>&09D}|2~BiMo=z4YEjXlWh)V&qs;*C{`UMxp$9 zX)QB?G$fPD6z5_pNs>Jeh{^&U^)Wbr?2D6-q?)`*1k@!UvwQgl8eG$r+)NnFoT)L6 zg7lEh+E6J17krfYJCSjWzm67hEth24pomhz71|Qodn#oAILN)*Vwu2qpJirG)4Wnv}9GWOFrQg%Je+gNrPl8mw7ykE8{ z=|B4+uwC&bpp%eFcRU6{mxRV32VeH8XxX>v$du<$(DfinaaWxP<+Y97Z#n#U~V zVEu-GoPD=9$}P;xv+S~Ob#mmi$JQmE;Iz4(){y*9pFyW-jjgdk#oG$fl4o9E8bo|L zWjo4l%n51@Kz-n%zeSCD`uB?T%FVk+KBI}=ve zvlcS#wt`U6wrJo}6I6Rwb=1GzZfwE=I&Ne@p7*pH84XShXYJRgvK)UjQL%R9Zbm(m zxzTQsLTON$WO7vM)*vl%Pc0JH7WhP;$z@j=y#avW4X8iqy6mEYr@-}PW?H)xfP6fQ z&tI$F{NNct4rRMSHhaelo<5kTYq+(?pY)Ieh8*sa83EQfMrFupMM@nfEV@EmdHUv9 z35uzIrIuo4#WnF^_jcpC@uNNaYTQ~uZWOE6P@LFT^1@$o&q+9Qr8YR+ObBkpP9=F+$s5+B!mX2~T zAuQ6RenX?O{IlLMl1%)OK{S7oL}X%;!XUxU~xJN8xk z`xywS*naF(J#?vOpB(K=o~lE;m$zhgPWDB@=p#dQIW>xe_p1OLoWInJRKbEuoncf; zmS1!u-ycc1qWnDg5Nk2D)BY%jmOwCLC+Ny>`f&UxFowIsHnOXfR^S;&F(KXd{ODlm z$6#1ccqt-HIH9)|@fHnrKudu!6B$_R{fbCIkSIb#aUN|3RM>zuO>dpMbROZ`^hvS@ z$FU-;e4W}!ubzKrU@R*dW*($tFZ>}dd*4_mv)#O>X{U@zSzQt*83l9mI zI$8O<5AIDx`wo0}f2fsPC_l>ONx_`E7kdXu{YIZbp1$(^oBAH({T~&oQ&1{X951QW zmhHUxd)t%GQ9#ak5fTjk-cahWC;>^Rg7(`TVlvy0W@Y!Jc%QL3Ozu# zDPIqBCy&T2PWBj+d-JA-pxZlM=9ja2ce|3B(^VCF+a*MMp`(rH>Rt6W1$;r{n1(VK zLs>UtkT43LR2G$AOYHVailiqk7naz2yZGLo*xQs!T9VN5Q>eE(w zw$4&)&6xIV$IO^>1N-jrEUg>O8G4^@y+-hQv6@OmF@gy^nL_n1P1-Rtyy$Bl;|VcV zF=p*&41-qI5gG9UhKmmnjs932!6hceXa#-qfK;3d*a{)BrwNFeKU|ge?N!;zk+kB! zMD_uHJR#%b54c2tr~uGPLTRLg$`fupo}cRJeTwK;~}A>(Acy4k-Xk&Aa1&eWYS1ULWUj@fhBiWY$pdfy+F z@G{OG{*v*mYtH3OdUjwEr6%_ZPZ3P{@rfbNPQG!BZ7lRyC^xlMpWH`@YRar`tr}d> z#wz87t?#2FsH-jM6m{U=gp6WPrZ%*w0bFm(T#7m#v^;f%Z!kCeB5oiF`W33W5Srdt zdU?YeOdPG@98H7NpI{(uN{FJdu14r(URPH^F6tOpXuhU7T9a{3G3_#Ldfx_nT(Hec zo<1dyhsVsTw;ZkVcJ_0-h-T3G1W@q)_Q30LNv)W?FbMH+XJ* zy=$@39Op|kZv`Rt>X`zg&at(?PO^I=X8d9&myFEx#S`dYTg1W+iE?vt#b47QwoHI9 zNP+|3WjtXo{u}VG(lLUaW0&@yD|O?4TS4dfJI`HC-^q;M(b3r2;7|FONXphw-%7~* z&;2!X17|05+kZOpQ3~3!Nb>O94b&ZSs%p)TK)n3m=4eiblVtSx@KNFgBY_xV6ts;NF;GcGxMP8OKV^h6LmSb2E#Qnw ze!6Mnz7>lE9u{AgQ~8u2zM8CYD5US8dMDX-5iMlgpE9m*s+Lh~A#P1er*rF}GHV3h z=`STo?kIXw8I<`W0^*@mB1$}pj60R{aJ7>C2m=oghKyxMbFNq#EVLgP0cH3q7H z%0?L93-z6|+jiN|@v>ix?tRBU(v-4RV`}cQH*fp|)vd3)8i9hJ3hkuh^8dz{F5-~_ zUUr1T3cP%cCaTooM8dj|4*M=e6flH0&8ve32Q)0dyisl))XkZ7Wg~N}6y`+Qi2l+e zUd#F!nJp{#KIjbQdI`%oZ`?h=5G^kZ_uN`<(`3;a!~EMsWV|j-o>c?x#;zR2ktiB! z);5rrHl?GPtr6-o!tYd|uK;Vbsp4P{v_4??=^a>>U4_aUXPWQ$FPLE4PK$T^3Gkf$ zHo&9$U&G`d(Os6xt1r?sg14n)G8HNyWa^q8#nf0lbr4A-Fi;q6t-`pAx1T*$eKM*$ z|CX|gDrk#&1}>5H+`EjV$9Bm)Njw&7-ZR{1!CJTaXuP!$Pcg69`{w5BRHysB$(tWUes@@6aM69kb|Lx$%BRY^-o6bjH#0!7b;5~{6J+jKxU!Kmi# zndh@+?}WKSRY2gZ?Q`{(Uj|kb1%VWmRryOH0T)f3cKtG4oIF=F7RaRnH0Rc_&372={_3lRNsr95%ZO{IX{p@YJ^EI%+gvvKes5cY+PE@unghjdY5#9A!G z70u6}?zmd?v+{`vCu-53_v5@z)X{oPC@P)iA3jK$`r zSA2a7&!^zmUiZ82R2=1cumBQwOJUPz5Ay`RLfY(EiwKkrx%@YN^^XuET;tE zmr-6~I7j!R!KrHu5CWGSChO6deaLWa*9LLJbcAJsFd%Dy>a!>J`N)Z&oiU4OEP-!Ti^_!p}O?7`}i7Lsf$-gBkuY*`Zb z7=!nTT;5z$_5$=J=Ko+Cp|Q0J=%oFr>hBgnL3!tvFoLNhf#D0O=X^h+x08iB;@8pXdRHxX}6R4k@i6%vmsQwu^5z zk1ip`#^N)^#Lg#HOW3sPI33xqFB4#bOPVnY%d6prwxf;Y-w9{ky4{O6&94Ra8VN@K zb-lY;&`HtxW@sF!doT5T$2&lIvJpbKGMuDAFM#!QPXW87>}=Q4J3JeXlwHys?!1^#37q_k?N@+u&Ns20pEoBeZC*np;i;M{2C0Z4_br2gsh6eL z#8`#sn41+$iD?^GL%5?cbRcaa-Nx0vE(D=*WY%rXy3B%gNz0l?#noGJGP728RMY#q z=2&aJf@DcR?QbMmN)ItUe+VM_U!ryqA@1VVt$^*xYt~-qvW!J4Tp<-3>jT=7Zow5M z8mSKp0v4b%a8bxFr>3MwZHSWD73D@+$5?nZAqGM#>H@`)mIeC#->B)P8T$zh-Pxnc z8)~Zx?TWF4(YfKuF3WN_ckpCe5;x4V4AA3(i$pm|78{%!q?|~*eH0f=?j6i)n~Hso zmTo>vqEtB)`%hP55INf7HM@taH)v`Fw40Ayc*R!T?O{ziUpYmP)AH`euTK!zg9*6Z z!>M=$3pd0!&TzU=hc_@@^Yd3eUQpX4-33}b{?~5t5lgW=ldJ@dUAH%`l5US1y_`40 zs(X`Qk}vvMDYYq+@Rm+~IyCX;iD~pMgq^KY)T*aBz@DYEB={PxA>)mI6tM*sx-DmGQHEaHwRrAmNjO!ZLHO4b;;5mf@zzlPhkP($JeZGE7 z?^XN}Gf_feGoG~BjUgVa*)O`>lX=$BSR2)uD<9 z>o^|nb1^oVDhQbfW>>!;8-7<}nL6L^V*4pB=>wwW+RXAeRvKED(n1;R`A6v$6gy0I(;Vf?!4;&sgn7F%LpM}6PQ?0%2Z@b{It<(G1CZ|>913E0nR2r^Pa*Bp z@tFGi*CQ~@Yc-?{cwu1 zsilf=k^+Qs>&WZG(3WDixisHpR>`+ihiRwkL(3T|=xsoNP*@XX3BU8hr57l3k;pni zI``=3Nl4xh4oDj<%>Q1zYXHr%Xg_xrK3Nq?vKX3|^Hb(Bj+lONTz>4yhU-UdXt2>j z<>S4NB&!iE+ao{0Tx^N*^|EZU;0kJkx@zh}S^P{ieQjGl468CbC`SWnwLRYYiStXm zOxt~Rb3D{dz=nHMcY)#r^kF8|q8KZHVb9FCX2m^X*(|L9FZg!5a7((!J8%MjT$#Fs)M1Pb zq6hBGp%O1A+&%2>l0mpaIzbo&jc^!oN^3zxap3V2dNj3x<=TwZ&0eKX5PIso9j1;e zwUg+C&}FJ`k(M|%%}p=6RPUq4sT3-Y;k-<68ciZ~_j|bt>&9ZLHNVrp#+pk}XvM{8 z`?k}o-!if>hVlCP9j%&WI2V`5SW)BCeR5>MQhF)po=p~AYN%cNa_BbV6EEh_kk^@a zD>4&>uCGCUmyA-c)%DIcF4R6!>?6T~Mj_m{Hpq`*(wj>foHL;;%;?(((YOxGt)Bhx zuS+K{{CUsaC++%}S6~CJ=|vr(iIs-je)e9uJEU8ZJAz)w166q)R^2XI?@E2vUQ!R% zn@dxS!JcOimXkWJBz8Y?2JKQr>`~SmE2F2SL38$SyR1^yqj8_mkBp)o$@+3BQ~Mid z9U$XVqxX3P=XCKj0*W>}L0~Em`(vG<>srF8+*kPrw z20{z(=^w+ybdGe~Oo_i|hYJ@kZl*(9sHw#Chi&OIc?w`nBODp?ia$uF%Hs(X>xm?j zqZQ`Ybf@g#wli`!-al~3GWiE$K+LCe=Ndi!#CVjzUZ z!sD2O*;d28zkl))m)YN7HDi^z5IuNo3^w(zy8 zszJG#mp#Cj)Q@E@r-=NP2FVxxEAeOI2e=|KshybNB6HgE^(r>HD{*}S}mO>LuRGJT{*tfTzw_#+er-0${}%YPe@CMJ1Ng#j#)i)SnY@ss3gL;g zg2D~#Kpdfu#G;q1qz_TwSz1VJT(b3zby$Vk&;Y#1(A)|xj`_?i5YQ;TR%jice5E;0 zYHg;`zS5{S*9xI6o^j>rE8Ua*XhIw{_-*&@(R|C(am8__>+Ws&Q^ymy*X4~hR2b5r zm^p3sw}yv=tdyncy_Ui7{BQS732et~Z_@{-IhHDXAV`(Wlay<#hb>%H%WDi+K$862nA@BDtM#UCKMu+kM`!JHyWSi?&)A7_ z3{cyNG%a~nnH_!+;g&JxEMAmh-Z}rC!o7>OVzW&PoMyTA_g{hqXG)SLraA^OP**<7 zjWbr7z!o2n3hnx7A=2O=WL;`@9N{vQIM@&|G-ljrPvIuJHYtss0Er0fT5cMXNUf1B z7FAwBDixt0X7C3S)mPe5g`YtME23wAnbU)+AtV}z+e8G;0BP=bI;?(#|Ep!vVfDbK zvx+|CKF>yt0hWQ3drchU#XBU+HiuG*V^snFAPUp-5<#R&BUAzoB!aZ+e*KIxa26V}s6?nBK(U-7REa573wg-jqCg>H8~>O{ z*C0JL-?X-k_y%hpUFL?I>0WV{oV`Nb)nZbJG01R~AG>flIJf)3O*oB2i8~;!P?Wo_ z0|QEB*fifiL6E6%>tlAYHm2cjTFE@*<);#>689Z6S#BySQ@VTMhf9vYQyLeDg1*F} zjq>i1*x>5|CGKN{l9br3kB0EHY|k4{%^t7-uhjd#NVipUZa=EUuE5kS1_~qYX?>hJ z$}!jc9$O$>J&wnu0SgfYods^z?J4X;X7c77Me0kS-dO_VUQ39T(Kv(Y#s}Qqz-0AH z^?WRL(4RzpkD+T5FG_0NyPq-a-B7A5LHOCqwObRJi&oRi(<;OuIN7SV5PeHU$<@Zh zPozEV`dYmu0Z&Tqd>t>8JVde9#Pt+l95iHe$4Xwfy1AhI zDM4XJ;bBTTvRFtW>E+GzkN)9k!hA5z;xUOL2 zq4}zn-DP{qc^i|Y%rvi|^5k-*8;JZ~9a;>-+q_EOX+p1Wz;>i7c}M6Nv`^NY&{J-> z`(mzDJDM}QPu5i44**2Qbo(XzZ-ZDu%6vm8w@DUarqXj41VqP~ zs&4Y8F^Waik3y1fQo`bVUH;b=!^QrWb)3Gl=QVKr+6sxc=ygauUG|cm?|X=;Q)kQ8 zM(xrICifa2p``I7>g2R~?a{hmw@{!NS5`VhH8+;cV(F>B94M*S;5#O`YzZH1Z%yD? zZ61w(M`#aS-*~Fj;x|J!KM|^o;MI#Xkh0ULJcA?o4u~f%Z^16ViA27FxU5GM*rKq( z7cS~MrZ=f>_OWx8j#-Q3%!aEU2hVuTu(7`TQk-Bi6*!<}0WQi;_FpO;fhpL4`DcWp zGOw9vx0N~6#}lz(r+dxIGZM3ah-8qrqMmeRh%{z@dbUD2w15*_4P?I~UZr^anP}DB zU9CCrNiy9I3~d#&!$DX9e?A});BjBtQ7oGAyoI$8YQrkLBIH@2;lt4E^)|d6Jwj}z z&2_E}Y;H#6I4<10d_&P0{4|EUacwFHauvrjAnAm6yeR#}f}Rk27CN)vhgRqEyPMMS7zvunj2?`f;%?alsJ+-K+IzjJx>h8 zu~m_y$!J5RWAh|C<6+uiCNsOKu)E72M3xKK(a9Okw3e_*O&}7llNV!=P87VM2DkAk zci!YXS2&=P0}Hx|wwSc9JP%m8dMJA*q&VFB0yMI@5vWoAGraygwn){R+Cj6B1a2Px z5)u(K5{+;z2n*_XD!+Auv#LJEM)(~Hx{$Yb^ldQmcYF2zNH1V30*)CN_|1$v2|`LnFUT$%-tO0Eg|c5$BB~yDfzS zcOXJ$wpzVK0MfTjBJ0b$r#_OvAJ3WRt+YOLlJPYMx~qp>^$$$h#bc|`g0pF-Ao43? z>*A+8lx>}L{p(Tni2Vvk)dtzg$hUKjSjXRagj)$h#8=KV>5s)J4vGtRn5kP|AXIz! zPgbbVxW{2o4s-UM;c#We8P&mPN|DW7_uLF!a|^0S=wr6Esx9Z$2|c1?GaupU6$tb| zY_KU`(_29O_%k(;>^|6*pZURH3`@%EuKS;Ns z1lujmf;r{qAN&Q0&m{wJSZ8MeE7RM5+Sq;ul_ z`+ADrd_Um+G37js6tKsArNB}n{p*zTUxQr>3@wA;{EUbjNjlNd6$Mx zg0|MyU)v`sa~tEY5$en7^PkC=S<2@!nEdG6L=h(vT__0F=S8Y&eM=hal#7eM(o^Lu z2?^;05&|CNliYrq6gUv;|i!(W{0N)LWd*@{2q*u)}u*> z7MQgk6t9OqqXMln?zoMAJcc zMKaof_Up})q#DzdF?w^%tTI7STI^@8=Wk#enR*)&%8yje>+tKvUYbW8UAPg55xb70 zEn5&Ba~NmOJlgI#iS8W3-@N%>V!#z-ZRwfPO1)dQdQkaHsiqG|~we2ALqG7Ruup(DqSOft2RFg_X%3w?6VqvV1uzX_@F(diNVp z4{I|}35=11u$;?|JFBEE*gb;T`dy+8gWJ9~pNsecrO`t#V9jW-6mnfO@ff9od}b(3s4>p0i30gbGIv~1@a^F2kl7YO;DxmF3? zWi-RoXhzRJV0&XE@ACc?+@6?)LQ2XNm4KfalMtsc%4!Fn0rl zpHTrHwR>t>7W?t!Yc{*-^xN%9P0cs0kr=`?bQ5T*oOo&VRRu+1chM!qj%2I!@+1XF z4GWJ=7ix9;Wa@xoZ0RP`NCWw0*8247Y4jIZ>GEW7zuoCFXl6xIvz$ezsWgKdVMBH> z{o!A7f;R-@eK9Vj7R40xx)T<2$?F2E<>Jy3F;;=Yt}WE59J!1WN367 zA^6pu_zLoZIf*x031CcwotS{L8bJE(<_F%j_KJ2P_IusaZXwN$&^t716W{M6X2r_~ zaiMwdISX7Y&Qi&Uh0upS3TyEIXNDICQlT5fHXC`aji-c{U(J@qh-mWl-uMN|T&435 z5)a1dvB|oe%b2mefc=Vpm0C%IUYYh7HI*;3UdgNIz}R##(#{(_>82|zB0L*1i4B5j-xi9O4x10rs_J6*gdRBX=@VJ+==sWb&_Qc6tSOowM{BX@(zawtjl zdU!F4OYw2@Tk1L^%~JCwb|e#3CC>srRHQ*(N%!7$Mu_sKh@|*XtR>)BmWw!;8-mq7 zBBnbjwx8Kyv|hd*`5}84flTHR1Y@@uqjG`UG+jN_YK&RYTt7DVwfEDXDW4U+iO{>K zw1hr{_XE*S*K9TzzUlJH2rh^hUm2v7_XjwTuYap|>zeEDY$HOq3X4Tz^X}E9z)x4F zs+T?Ed+Hj<#jY-`Va~fT2C$=qFT-5q$@p9~0{G&eeL~tiIAHXA!f6C(rAlS^)&k<- zXU|ZVs}XQ>s5iONo~t!XXZgtaP$Iau;JT%h)>}v54yut~pykaNye4axEK#5@?TSsQ zE;Jvf9I$GVb|S`7$pG)4vgo9NXsKr?u=F!GnA%VS2z$@Z(!MR9?EPcAqi5ft)Iz6sNl`%kj+_H-X`R<>BFrBW=fSlD|{`D%@Rcbu2?%>t7i34k?Ujb)2@J-`j#4 zLK<69qcUuniIan-$A1+fR=?@+thwDIXtF1Tks@Br-xY zfB+zblrR(ke`U;6U~-;p1Kg8Lh6v~LjW@9l2P6s+?$2!ZRPX`(ZkRGe7~q(4&gEi<$ch`5kQ?*1=GSqkeV z{SA1EaW_A!t{@^UY2D^YO0(H@+kFVzZaAh0_`A`f(}G~EP~?B|%gtxu&g%^x{EYSz zk+T;_c@d;+n@$<>V%P=nk36?L!}?*=vK4>nJSm+1%a}9UlmTJTrfX4{Lb7smNQn@T zw9p2%(Zjl^bWGo1;DuMHN(djsEm)P8mEC2sL@KyPjwD@d%QnZ$ zMJ3cnn!_!iP{MzWk%PI&D?m?C(y2d|2VChluN^yHya(b`h>~GkI1y;}O_E57zOs!{ zt2C@M$^PR2U#(dZmA-sNreB@z-yb0Bf7j*yONhZG=onhx>t4)RB`r6&TP$n zgmN*)eCqvgriBO-abHQ8ECN0bw?z5Bxpx z=jF@?zFdVn?@gD5egM4o$m`}lV(CWrOKKq(sv*`mNcHcvw&Xryfw<{ch{O&qc#WCTXX6=#{MV@q#iHYba!OUY+MGeNTjP%Fj!WgM&`&RlI^=AWTOqy-o zHo9YFt!gQ*p7{Fl86>#-JLZo(b^O`LdFK~OsZBRR@6P?ad^Ujbqm_j^XycM4ZHFyg ziUbIFW#2tj`65~#2V!4z7DM8Z;fG0|APaQ{a2VNYpNotB7eZ5kp+tPDz&Lqs0j%Y4tA*URpcfi z_M(FD=fRGdqf430j}1z`O0I=;tLu81bwJXdYiN7_&a-?ly|-j*+=--XGvCq#32Gh(=|qj5F?kmihk{%M&$}udW5)DHK zF_>}5R8&&API}o0osZJRL3n~>76nUZ&L&iy^s>PMnNcYZ|9*1$v-bzbT3rpWsJ+y{ zPrg>5Zlery96Um?lc6L|)}&{992{_$J&=4%nRp9BAC6!IB=A&=tF>r8S*O-=!G(_( zwXbX_rGZgeiK*&n5E;f=k{ktyA1(;x_kiMEt0*gpp_4&(twlS2e5C?NoD{n>X2AT# zY@Zp?#!b1zNq96MQqeO*M1MMBin5v#RH52&Xd~DO6-BZLnA6xO1$sou(YJ1Dlc{WF zVa%2DyYm`V#81jP@70IJ;DX@y*iUt$MLm)ByAD$eUuji|5{ptFYq(q)mE(5bOpxjM z^Q`AHWq44SG3`_LxC9fwR)XRVIp=B%<(-lOC3jI#bb@dK(*vjom!=t|#<@dZql%>O z15y^{4tQoeW9Lu%G&V$90x6F)xN6y_oIn;!Q zs)8jT$;&;u%Y>=T3hg34A-+Y*na=|glcStr5D;&5*t5*DmD~x;zQAV5{}Ya`?RRGa zT*t9@$a~!co;pD^!J5bo?lDOWFx%)Y=-fJ+PDGc0>;=q=s?P4aHForSB+)v0WY2JH z?*`O;RHum6j%#LG)Vu#ciO#+jRC3!>T(9fr+XE7T2B7Z|0nR5jw@WG)kDDzTJ=o4~ zUpeyt7}_nd`t}j9BKqryOha{34erm)RmST)_9Aw)@ zHbiyg5n&E{_CQR@h<}34d7WM{s{%5wdty1l+KX8*?+-YkNK2Be*6&jc>@{Fd;Ps|| z26LqdI3#9le?;}risDq$K5G3yoqK}C^@-8z^wj%tdgw-6@F#Ju{Sg7+y)L?)U$ez> zoOaP$UFZ?y5BiFycir*pnaAaY+|%1%8&|(@VB)zweR%?IidwJyK5J!STzw&2RFx zZV@qeaCB01Hu#U9|1#=Msc8Pgz5P*4Lrp!Q+~(G!OiNR{qa7|r^H?FC6gVhkk3y7=uW#Sh;&>78bZ}aK*C#NH$9rX@M3f{nckYI+5QG?Aj1DM)@~z_ zw!UAD@gedTlePB*%4+55naJ8ak_;))#S;4ji!LOqY5VRI){GMwHR~}6t4g>5C_#U# ztYC!tjKjrKvRy=GAsJVK++~$|+s!w9z3H4G^mACv=EErXNSmH7qN}%PKcN|8%9=i)qS5+$L zu&ya~HW%RMVJi4T^pv?>mw*Gf<)-7gf#Qj|e#w2|v4#t!%Jk{&xlf;$_?jW*n!Pyx zkG$<18kiLOAUPuFfyu-EfWX%4jYnjBYc~~*9JEz6oa)_R|8wjZA|RNrAp%}14L7fW zi7A5Wym*K+V8pkqqO-X#3ft{0qs?KVt^)?kS>AicmeO&q+~J~ zp0YJ_P~_a8j= zsAs~G=8F=M{4GZL{|B__UorX@MRNQLn?*_gym4aW(~+i13knnk1P=khoC-ViMZk+x zLW(l}oAg1H`dU+Fv**;qw|ANDSRs>cGqL!Yw^`; zv;{E&8CNJcc)GHzTYM}f&NPw<6j{C3gaeelU#y!M)w-utYEHOCCJo|Vgp7K6C_$14 zqIrLUB0bsgz^D%V%fbo2f9#yb#CntTX?55Xy|Kps&Xek*4_r=KDZ z+`TQuv|$l}MWLzA5Ay6Cvsa^7xvwXpy?`w(6vx4XJ zWuf1bVSb#U8{xlY4+wlZ$9jjPk)X_;NFMqdgq>m&W=!KtP+6NL57`AMljW+es zzqjUjgz;V*kktJI?!NOg^s_)ph45>4UDA!Vo0hn>KZ+h-3=?Y3*R=#!fOX zP$Y~+14$f66ix?UWB_6r#fMcC^~X4R-<&OD1CSDNuX~y^YwJ>sW0j`T<2+3F9>cLo z#!j57$ll2K9(%$4>eA7(>FJX5e)pR5&EZK!IMQzOfik#FU*o*LGz~7u(8}XzIQRy- z!U7AlMTIe|DgQFmc%cHy_9^{o`eD%ja_L>ckU6$O4*U**o5uR7`FzqkU8k4gxtI=o z^P^oGFPm5jwZMI{;nH}$?p@uV8FT4r=|#GziKXK07bHJLtK}X%I0TON$uj(iJ`SY^ zc$b2CoxCQ>7LH@nxcdW&_C#fMYBtTxcg46dL{vf%EFCZ~eErMvZq&Z%Lhumnkn^4A zsx$ay(FnN7kYah}tZ@0?-0Niroa~13`?hVi6`ndno`G+E8;$<6^gsE-K3)TxyoJ4M zb6pj5=I8^FD5H@`^V#Qb2^0cx7wUz&cruA5g>6>qR5)O^t1(-qqP&1g=qvY#s&{bx zq8Hc%LsbK1*%n|Y=FfojpE;w~)G0-X4i*K3{o|J7`krhIOd*c*$y{WIKz2n2*EXEH zT{oml3Th5k*vkswuFXdGDlcLj15Nec5pFfZ*0?XHaF_lVuiB%Pv&p7z)%38}%$Gup zVTa~C8=cw%6BKn_|4E?bPNW4PT7}jZQLhDJhvf4z;~L)506IE0 zX!tWXX(QOQPRj-p80QG79t8T2^az4Zp2hOHziQlvT!|H)jv{Ixodabzv6lBj)6WRB z{)Kg@$~~(7$-az?lw$4@L%I&DI0Lo)PEJJziWP33a3azb?jyXt1v0N>2kxwA6b%l> zZqRpAo)Npi&loWbjFWtEV)783BbeIAhqyuc+~>i7aQ8shIXt)bjCWT6$~ro^>99G} z2XfmT0(|l!)XJb^E!#3z4oEGIsL(xd; zYX1`1I(cG|u#4R4T&C|m*9KB1`UzKvho5R@1eYtUL9B72{i(ir&ls8g!pD ztR|25xGaF!4z5M+U@@lQf(12?xGy`!|3E}7pI$k`jOIFjiDr{tqf0va&3pOn6Pu)% z@xtG2zjYuJXrV)DUrIF*y<1O1<$#54kZ#2;=X51J^F#0nZ0(;S$OZDt_U2bx{RZ=Q zMMdd$fH|!s{ zXq#l;{`xfV`gp&C>A`WrQU?d{!Ey5(1u*VLJt>i27aZ-^&2IIk=zP5p+{$q(K?2(b z8?9h)kvj9SF!Dr zoyF}?V|9;6abHxWk2cEvGs$-}Pg}D+ZzgkaN&$Snp%;5m%zh1E#?Wac-}x?BYlGN#U#Mek*}kek#I9XaHt?mz3*fDrRTQ#&#~xyeqJk1QJ~E$7qsw6 z?sV;|?*=-{M<1+hXoj?@-$y+(^BJ1H~wQ9G8C0#^aEAyhDduNX@haoa=PuPp zYsGv8UBfQaRHgBgLjmP^eh>fLMeh{8ic)?xz?#3kX-D#Z{;W#cd_`9OMFIaJg-=t`_3*!YDgtNQ2+QUEAJB9M{~AvT$H`E)IKmCR21H532+ata8_i_MR@ z2Xj<3w<`isF~Ah$W{|9;51ub*f4#9ziKrOR&jM{x7I_7()O@`F*5o$KtZ?fxU~g`t zUovNEVKYn$U~VX8eR)qb`7;D8pn*Pp$(otYTqL)5KH$lUS-jf}PGBjy$weoceAcPp z&5ZYB$r&P$MN{0H0AxCe4Qmd3T%M*5d4i%#!nmBCN-WU-4m4Tjxn-%j3HagwTxCZ9 z)j5vO-C7%s%D!&UfO>bi2oXiCw<-w{vVTK^rVbv#W=WjdADJy8$khnU!`ZWCIU`># zyjc^1W~pcu>@lDZ{zr6gv%)2X4n27~Ve+cQqcND%0?IFSP4sH#yIaXXYAq^z3|cg` z`I3$m%jra>e2W-=DiD@84T!cb%||k)nPmEE09NC%@PS_OLhkrX*U!cgD*;;&gIaA(DyVT4QD+q_xu z>r`tg{hiGY&DvD-)B*h+YEd+Zn)WylQl}<4>(_NlsKXCRV;a)Rcw!wtelM2_rWX`j zTh5A|i6=2BA(iMCnj_fob@*eA;V?oa4Z1kRBGaU07O70fb6-qmA$Hg$ps@^ka1=RO zTbE_2#)1bndC3VuK@e!Sftxq4=Uux}fDxXE#Q5_x=E1h>T5`DPHz zbH<_OjWx$wy7=%0!mo*qH*7N4tySm+R0~(rbus`7;+wGh;C0O%x~fEMkt!eV>U$`i z5>Q(o z=t$gPjgGh0&I7KY#k50V7DJRX<%^X z>6+ebc9efB3@eE2Tr){;?_w`vhgF>`-GDY(YkR{9RH(MiCnyRtd!LxXJ75z+?2 zGi@m^+2hKJ5sB1@Xi@s_@p_Kwbc<*LQ_`mr^Y%j}(sV_$`J(?_FWP)4NW*BIL~sR>t6 zM;qTJZ~GoY36&{h-Pf}L#y2UtR}>ZaI%A6VkU>vG4~}9^i$5WP2Tj?Cc}5oQxe2=q z8BeLa$hwCg_psjZyC2+?yX4*hJ58Wu^w9}}7X*+i5Rjqu5^@GzXiw#SUir1G1`jY% zOL=GE_ENYxhcyUrEt9XlMNP6kx6h&%6^u3@zB8KUCAa18T(R2J`%JjWZ z!{7cXaEW+Qu*iJPu+m>QqW}Lo$4Z+!I)0JNzZ&_M%=|B1yejFRM04bGAvu{=lNPd+ zJRI^DRQ(?FcVUD+bgEcAi@o(msqys9RTCG#)TjI!9~3-dc`>gW;HSJuQvH~d`MQs86R$|SKXHh zqS9Qy)u;T`>>a!$LuaE2keJV%;8g)tr&Nnc;EkvA-RanHXsy)D@XN0a>h}z2j81R; zsUNJf&g&rKpuD0WD@=dDrPHdBoK42WoBU|nMo17o(5^;M|dB4?|FsAGVrSyWcI`+FVw^vTVC`y}f(BwJl zrw3Sp151^9=}B})6@H*i4-dIN_o^br+BkcLa^H56|^2XsT0dESw2 zMX>(KqNl=x2K5=zIKg}2JpGAZu{I_IO}0$EQ5P{4zol**PCt3F4`GX}2@vr8#Y)~J zKb)gJeHcFnR@4SSh%b;c%J`l=W*40UPjF#q{<}ywv-=vHRFmDjv)NtmC zQx9qm)d%0zH&qG7AFa3VAU1S^(n8VFTC~Hb+HjYMjX8r#&_0MzlNR*mnLH5hi}`@{ zK$8qiDDvS_(L9_2vHgzEQ${DYSE;DqB!g*jhJghE&=LTnbgl&Xepo<*uRtV{2wDHN z)l;Kg$TA>Y|K8Lc&LjWGj<+bp4Hiye_@BfU(y#nF{fpR&|Ltbye?e^j0}8JC4#xi% zv29ZR%8%hk=3ZDvO-@1u8KmQ@6p%E|dlHuy#H1&MiC<*$YdLkHmR#F3ae;bKd;@*i z2_VfELG=B}JMLCO-6UQy^>RDE%K4b>c%9ki`f~Z2Qu8hO7C#t%Aeg8E%+}6P7Twtg z-)dj(w}_zFK&86KR@q9MHicUAucLVshUdmz_2@32(V`y3`&Kf8Q2I)+!n0mR=rrDU zXvv^$ho;yh*kNqJ#r1}b0|i|xRUF6;lhx$M*uG3SNLUTC@|htC z-=fsw^F%$qqz4%QdjBrS+ov}Qv!z00E+JWas>p?z@=t!WWU3K*?Z(0meTuTOC7OTx zU|kFLE0bLZ+WGcL$u4E}5dB0g`h|uwv3=H6f+{5z9oLv-=Q45+n~V4WwgO=CabjM% zBAN+RjM65(-}>Q2V#i1Na@a0`08g&y;W#@sBiX6Tpy8r}*+{RnyGUT`?XeHSqo#|J z^ww~c;ou|iyzpErDtlVU=`8N7JSu>4M z_pr9=tX0edVn9B}YFO2y(88j#S{w%E8vVOpAboK*27a7e4Ekjt0)hIX99*1oE;vex z7#%jhY=bPijA=Ce@9rRO(Vl_vnd00!^TAc<+wVvRM9{;hP*rqEL_(RzfK$er_^SN; z)1a8vo8~Dr5?;0X0J62Cusw$A*c^Sx1)dom`-)Pl7hsW4i(r*^Mw`z5K>!2ixB_mu z*Ddqjh}zceRFdmuX1akM1$3>G=#~|y?eYv(e-`Qy?bRHIq=fMaN~fB zUa6I8Rt=)jnplP>yuS+P&PxeWpJ#1$F`iqRl|jF$WL_aZFZl@kLo&d$VJtu&w?Q0O zzuXK>6gmygq(yXJy0C1SL}T8AplK|AGNUOhzlGeK_oo|haD@)5PxF}rV+5`-w{Aag zus45t=FU*{LguJ11Sr-28EZkq;!mJO7AQGih1L4rEyUmp>B!%X0YemsrV3QFvlgt* z5kwlPzaiJ+kZ^PMd-RRbl(Y?F*m`4*UIhIuf#8q>H_M=fM*L_Op-<_r zBZagV=4B|EW+KTja?srADTZXCd3Yv%^Chfpi)cg{ED${SI>InNpRj5!euKv?=Xn92 zsS&FH(*w`qLIy$doc>RE&A5R?u zzkl1sxX|{*fLpXvIW>9d<$ePROttn3oc6R!sN{&Y+>Jr@yeQN$sFR z;w6A<2-0%UA?c8Qf;sX7>>uKRBv3Ni)E9pI{uVzX|6Bb0U)`lhLE3hK58ivfRs1}d zNjlGK0hdq0qjV@q1qI%ZFMLgcpWSY~mB^LK)4GZ^h_@H+3?dAe_a~k*;9P_d7%NEFP6+ zgV(oGr*?W(ql?6SQ~`lUsjLb%MbfC4V$)1E0Y_b|OIYxz4?O|!kRb?BGrgiH5+(>s zoqM}v*;OBfg-D1l`M6T6{K`LG+0dJ1)!??G5g(2*vlNkm%Q(MPABT$r13q?|+kL4- zf)Mi5r$sn;u41aK(K#!m+goyd$c!KPl~-&-({j#D4^7hQkV3W|&>l_b!}!z?4($OA z5IrkfuT#F&S1(`?modY&I40%gtroig{YMvF{K{>5u^I51k8RriGd${z)=5k2tG zM|&Bp5kDTfb#vfuTTd?)a=>bX=lokw^y9+2LS?kwHQIWI~pYgy7 zb?A-RKVm_vM5!9?C%qYdfRAw& zAU7`up~%g=p@}pg#b7E)BFYx3g%(J36Nw(Dij!b>cMl@CSNbrW!DBDbTD4OXk!G4x zi}JBKc8HBYx$J~31PXH+4^x|UxK~(<@I;^3pWN$E=sYma@JP|8YL`L(zI6Y#c%Q{6 z*APf`DU$S4pr#_!60BH$FGViP14iJmbrzSrOkR;f3YZa{#E7Wpd@^4E-zH8EgPc-# zKWFPvh%WbqU_%ZEt`=Q?odKHc7@SUmY{GK`?40VuL~o)bS|is$Hn=<=KGHOsEC5tB zFb|q}gGlL97NUf$G$>^1b^3E18PZ~Pm9kX%*ftnolljiEt@2#F2R5ah$zbXd%V_Ev zyDd{1o_uuoBga$fB@Fw!V5F3jIr=a-ykqrK?WWZ#a(bglI_-8pq74RK*KfQ z0~Dzus7_l;pMJYf>Bk`)`S8gF!To-BdMnVw5M-pyu+aCiC5dwNH|6fgRsIKZcF&)g zr}1|?VOp}I3)IR@m1&HX1~#wsS!4iYqES zK}4J{Ei>;e3>LB#Oly>EZkW14^@YmpbgxCDi#0RgdM${&wxR+LiX}B+iRioOB0(pDKpVEI;ND?wNx>%e|m{RsqR_{(nmQ z3ZS}@t!p4a(BKx_-CYwrcyJ5u1TO9bcXti$8sy>xcLKqKCc#~UOZYD{llKTSFEjJ~ zyNWt>tLU}*>^`TvPxtP%F`ZJQw@W0^>x;!^@?k_)9#bF$j0)S3;mH-IR5y82l|%=F z2lR8zhP?XNP-ucZZ6A+o$xOyF!w;RaLHGh57GZ|TCXhJqY~GCh)aXEV$1O&$c}La1 zjuJxkY9SM4av^Hb;i7efiYaMwI%jGy`3NdY)+mcJhF(3XEiSlU3c|jMBi|;m-c?~T z+x0_@;SxcoY=(6xNgO$bBt~Pj8`-<1S|;Bsjrzw3@zSjt^JC3X3*$HI79i~!$RmTz zsblZsLYs7L$|=1CB$8qS!tXrWs!F@BVuh?kN(PvE5Av-*r^iYu+L^j^m9JG^#=m>@ z=1soa)H*w6KzoR$B8mBCXoU;f5^bVuwQ3~2LKg!yxomG1#XPmn(?YH@E~_ED+W6mxs%x{%Z<$pW`~ON1~2XjP5v(0{C{+6Dm$00tsd3w=f=ZENy zOgb-=f}|Hb*LQ$YdWg<(u7x3`PKF)B7ZfZ6;1FrNM63 z?O6tE%EiU@6%rVuwIQjvGtOofZBGZT1Sh(xLIYt9c4VI8`!=UJd2BfLjdRI#SbVAX ziT(f*RI^T!IL5Ac>ql7uduF#nuCRJ1)2bdvAyMxp-5^Ww5p#X{rb5)(X|fEhDHHW{ zw(Lfc$g;+Q`B0AiPGtmK%*aWfQQ$d!*U<|-@n2HZvCWSiw^I>#vh+LyC;aaVWGbmkENr z&kl*8o^_FW$T?rDYLO1Pyi%>@&kJKQoH2E0F`HjcN}Zlnx1ddoDA>G4Xu_jyp6vuT zPvC}pT&Owx+qB`zUeR|4G;OH(<<^_bzkjln0k40t`PQxc$7h(T8Ya~X+9gDc8Z9{Z z&y0RAU}#_kQGrM;__MK9vwIwK^aoqFhk~dK!ARf1zJqHMxF2?7-8|~yoO@_~Ed;_wvT%Vs{9RK$6uUQ|&@#6vyBsFK9eZW1Ft#D2)VpQRwpR(;x^ zdoTgMqfF9iBl%{`QDv7B0~8{8`8k`C4@cbZAXBu00v#kYl!#_Wug{)2PwD5cNp?K^ z9+|d-4z|gZ!L{57>!Ogfbzchm>J1)Y%?NThxIS8frAw@z>Zb9v%3_3~F@<=LG%r*U zaTov}{{^z~SeX!qgSYow`_5)ij*QtGp4lvF`aIGQ>@3ZTkDmsl#@^5*NGjOuu82}o zzLF~Q9SW+mP=>88%eSA1W4_W7-Q>rdq^?t=m6}^tDPaBRGFLg%ak93W!kOp#EO{6& zP%}Iff5HZQ9VW$~+9r=|Quj#z*=YwcnssS~9|ub2>v|u1JXP47vZ1&L1O%Z1DsOrDfSIMHU{VT>&>H=9}G3i@2rP+rx@eU@uE8rJNec zij~#FmuEBj03F1~ct@C@$>y)zB+tVyjV3*n`mtAhIM0$58vM9jOQC}JJOem|EpwqeMuYPxu3sv}oMS?S#o6GGK@8PN59)m&K4Dc&X% z(;XL_kKeYkafzS3Wn5DD>Yiw{LACy_#jY4op(>9q>>-*9@C0M+=b#bknAWZ37^(Ij zq>H%<@>o4a#6NydoF{_M4i4zB_KG)#PSye9bk0Ou8h%1Dtl7Q_y#7*n%g)?m>xF~( zjqvOwC;*qvN_3(*a+w2|ao0D?@okOvg8JskUw(l7n`0fncglavwKd?~l_ryKJ^Ky! zKCHkIC-o7%fFvPa$)YNh022lakMar^dgL=t#@XLyNHHw!b?%WlM)R@^!)I!smZL@k zBi=6wE5)2v&!UNV(&)oOYW(6Qa!nUjDKKBf-~Da=#^HE4(@mWk)LPvhyN3i4goB$3K8iV7uh zsv+a?#c4&NWeK(3AH;ETrMOIFgu{_@%XRwCZ;L=^8Ts)hix4Pf3yJRQ<8xb^CkdmC z?c_gB)XmRsk`9ch#tx4*hO=#qS7={~Vb4*tTf<5P%*-XMfUUYkI9T1cEF;ObfxxI-yNuA=I$dCtz3ey znVkctYD*`fUuZ(57+^B*R=Q}~{1z#2!ca?)+YsRQb+lt^LmEvZt_`=j^wqig+wz@n@ z`LIMQJT3bxMzuKg8EGBU+Q-6cs5(@5W?N>JpZL{$9VF)veF`L5%DSYTNQEypW%6$u zm_~}T{HeHj1bAlKl8ii92l9~$dm=UM21kLemA&b$;^!wB7#IKWGnF$TVq!!lBlG4 z{?Rjz?P(uvid+|i$VH?`-C&Gcb3{(~Vpg`w+O);Wk1|Mrjxrht0GfRUnZqz2MhrXa zqgVC9nemD5)H$to=~hp)c=l9?#~Z_7i~=U-`FZxb-|TR9@YCxx;Zjo-WpMNOn2)z) zFPGGVl%3N$f`gp$gPnWC+f4(rmts%fidpo^BJx72zAd7|*Xi{2VXmbOm)1`w^tm9% znM=0Fg4bDxH5PxPEm{P3#A(mxqlM7SIARP?|2&+c7qmU8kP&iApzL|F>Dz)Ixp_`O zP%xrP1M6@oYhgo$ZWwrAsYLa4 z|I;DAvJxno9HkQrhLPQk-8}=De{9U3U%)dJ$955?_AOms!9gia%)0E$Mp}$+0er@< zq7J&_SzvShM?e%V?_zUu{niL@gt5UFOjFJUJ}L?$f%eU%jUSoujr{^O=?=^{19`ON zlRIy8Uo_nqcPa6@yyz`CM?pMJ^^SN^Fqtt`GQ8Q#W4kE7`V9^LT}j#pMChl!j#g#J zr-=CCaV%xyFeQ9SK+mG(cTwW*)xa(eK;_Z(jy)woZp~> zA(4}-&VH+TEeLzPTqw&FOoK(ZjD~m{KW05fiGLe@E3Z2`rLukIDahE*`u!ubU)9`o zn^-lyht#E#-dt~S>}4y$-mSbR8{T@}22cn^refuQ08NjLOv?JiEWjyOnzk<^R5%gO zhUH_B{oz~u#IYwVnUg8?3P*#DqD8#X;%q%HY**=I>>-S|!X*-!x1{^l#OnR56O>iD zc;i;KS+t$koh)E3)w0OjWJl_aW2;xF=9D9Kr>)(5}4FqUbk# zI#$N8o0w;IChL49m9CJTzoC!|u{Ljd%ECgBOf$}&jA^$(V#P#~)`&g`H8E{uv52pp zwto`xUL-L&WTAVREEm$0g_gYPL(^vHq(*t1WCH_6alhkeW&GCZ3hL)|{O-jiFOBrF z!EW=Jej|dqQitT6!B-7&io2K)WIm~Q)v@yq%U|VpV+I?{y0@Yd%n8~-NuuM*pM~KA z85YB};IS~M(c<}4Hxx>qRK0cdl&e?t253N%vefkgds>Ubn8X}j6Vpgs>a#nFq$osY z1ZRwLqFv=+BTb=i%D2Wv>_yE0z}+niZ4?rE|*a3d7^kndWGwnFqt+iZ(7+aln<}jzbAQ(#Z2SS}3S$%Bd}^ zc9ghB%O)Z_mTZMRC&H#)I#fiLuIkGa^`4e~9oM5zKPx?zjkC&Xy0~r{;S?FS%c7w< zWbMpzc(xSw?9tGxG~_l}Acq}zjt5ClaB7-!vzqnlrX;}$#+PyQ9oU)_DfePh2E1<7 ztok6g6K^k^DuHR*iJ?jw?bs_whk|bx`dxu^nC6#e{1*m~z1eq7m}Cf$*^Eua(oi_I zAL+3opNhJteu&mWQ@kQWPucmiP)4|nFG`b2tpC;h{-PI@`+h?9v=9mn|0R-n8#t=+Z*FD(c5 zjj79Jxkgck*DV=wpFgRZuwr%}KTm+dx?RT@aUHJdaX-ODh~gByS?WGx&czAkvkg;x zrf92l8$Or_zOwJVwh>5rB`Q5_5}ef6DjS*$x30nZbuO3dijS*wvNEqTY5p1_A0gWr znH<(Qvb!os14|R)n2Ost>jS2;d1zyLHu`Svm|&dZD+PpP{Bh>U&`Md;gRl64q;>{8MJJM$?UNUd`aC>BiLe>*{ zJY15->yW+<3rLgYeTruFDtk1ovU<$(_y7#HgUq>)r0{^}Xbth}V#6?%5jeFYt;SG^ z3qF)=uWRU;Jj)Q}cpY8-H+l_n$2$6{ZR?&*IGr{>ek!69ZH0ZoJ*Ji+ezzlJ^%qL3 zO5a`6gwFw(moEzqxh=yJ9M1FTn!eo&qD#y5AZXErHs%22?A+JmS&GIolml!)rZTnUDM3YgzYfT#;OXn)`PWv3Ta z!-i|-Wojv*k&bC}_JJDjiAK(Ba|YZgUI{f}TdEOFT2+}nPmttytw7j%@bQZDV1vvj z^rp{gRkCDmYJHGrE1~e~AE!-&6B6`7UxVQuvRrfdFkGX8H~SNP_X4EodVd;lXd^>eV1jN+Tt4}Rsn)R0LxBz0c=NXU|pUe!MQQFkGBWbR3&(jLm z%RSLc#p}5_dO{GD=DEFr=Fc% z85CBF>*t!6ugI?soX(*JNxBp+-DdZ4X0LldiK}+WWGvXV(C(Ht|!3$psR=&c*HIM=BmX;pRIpz@Ale{9dhGe(U2|Giv;# zOc|;?p67J=Q(kamB*aus=|XP|m{jN^6@V*Bpm?ye56Njh#vyJqE=DweC;?Rv7faX~ zde03n^I~0B2vUmr;w^X37tVxUK?4}ifsSH5_kpKZIzpYu0;Kv}SBGfI2AKNp+VN#z`nI{UNDRbo-wqa4NEls zICRJpu)??cj^*WcZ^MAv+;bDbh~gpN$1Cor<{Y2oyIDws^JsfW^5AL$azE(T0p&pP z1Mv~6Q44R&RHoH95&OuGx2srIr<@zYJTOMKiVs;Bx3py89I87LOb@%mr`0)#;7_~Z zzcZj8?w=)>%5@HoCHE_&hnu(n_yQ-L(~VjpjjkbT7e)Dk5??fApg(d>vwLRJ-x{um z*Nt?DqTSxh_MIyogY!vf1mU1`Gld-&L)*43f6dilz`Q@HEz;+>MDDYv9u!s;WXeao zUq=TaL$P*IFgJzrGc>j1dDOd zed+=ZBo?w4mr$2)Ya}?vedDopomhW1`#P<%YOJ_j=WwClX0xJH-f@s?^tmzs_j7t!k zK@j^zS0Q|mM4tVP5Ram$VbS6|YDY&y?Q1r1joe9dj08#CM{RSMTU}(RCh`hp_Rkl- zGd|Cv~G@F{DLhCizAm9AN!^{rNs8hu!G@8RpnGx7e`-+K$ffN<0qjR zGq^$dj_Tv!n*?zOSyk5skI7JVKJ)3jysnjIu-@VSzQiP8r6MzudCU=~?v-U8yzo^7 zGf~SUTvEp+S*!X9uX!sq=o}lH;r{pzk~M*VA(uyQ`3C8!{C;)&6)95fv(cK!%Cuz$ z_Zal57H6kPN>25KNiI6z6F)jzEkh#%OqU#-__Xzy)KyH};81#N6OfX$$IXWzOn`Q& z4f$Z1t>)8&8PcYfEwY5UadU1yg+U*(1m2ZlHoC-!2?gB!!fLhmTl))D@dhvkx#+Yj z1O=LV{(T%{^IeCuFK>%QR!VZ4GnO5tK8a+thWE zg4VytZrwcS?7^ zuZfhYnB8dwd%VLO?DK7pV5Wi<(`~DYqOXn8#jUIL^)12*Dbhk4GmL_E2`WX&iT16o zk(t|hok(Y|v-wzn?4x34T)|+SfZP>fiq!><*%vnxGN~ypST-FtC+@TPv*vYv@iU!_ z@2gf|PrgQ?Ktf*9^CnJ(x*CtZVB8!OBfg0%!wL;Z8(tYYre0vcnPGlyCc$V(Ipl*P z_(J!a=o@vp^%Efme!K74(Ke7A>Y}|sxV+JL^aYa{~m%5#$$+R1? zGaQhZTTX!#s#=Xtpegqero$RNt&`4xn3g$)=y*;=N=Qai)}~`xtxI_N*#MMCIq#HFifT zz(-*m;pVH&+4bixL&Bbg)W5FN^bH87pAHp)zPkWNMfTFqS=l~AC$3FX3kQUSh_C?-ZftyClgM)o_D7cX$RGlEYblux0jv5 zTr|i-I3@ZPCGheCl~BGhImF)K4!9@?pC(gi3ozX=a!|r1)LFxy_8c&wY0<^{2cm|P zv6Y`QktY*;I)IUd5y3ne1CqpVanlY45z8hf4&$EUBnucDj16pDa4&GI&TArYhf*xh zdj>*%APH8(h~c>o@l#%T>R$e>rwVx_WUB|~V`p^JHsg*y12lzj&zF}w6W09HwB2yb z%Q~`es&(;7#*DUC_w-Dmt7|$*?TA_m;zB+-u{2;Bg{O}nV7G_@7~<)Bv8fH^G$XG8$(&{A zwXJK5LRK%M34(t$&NI~MHT{UQ9qN-V_yn|%PqC81EIiSzmMM=2zb`mIwiP_b)x+2M z7Gd`83h79j#SItpQ}luuf2uOU`my_rY5T{6P#BNlb%h%<#MZb=m@y5aW;#o1^2Z)SWo+b`y0gV^iRcZtz5!-05vF z7wNo=hc6h4hc&s@uL^jqRvD6thVYtbErDK9k!;+a0xoE0WL7zLixjn5;$fXvT=O3I zT6jI&^A7k6R{&5#lVjz#8%_RiAa2{di{`kx79K+j72$H(!ass|B%@l%KeeKchYLe_ z>!(JC2fxsv>XVen+Y42GeYPxMWqm`6F$(E<6^s|g(slNk!lL*6v^W2>f6hh^mE$s= z3D$)}{V5(Qm&A6bp%2Q}*GZ5Qrf}n7*Hr51?bJOyA-?B4vg6y_EX<*-e20h{=0Mxs zbuQGZ$fLyO5v$nQ&^kuH+mNq9O#MWSfThtH|0q1i!NrWj^S}_P;Q1OkYLW6U^?_7G zx2wg?CULj7))QU(n{$0JE%1t2dWrMi2g-Os{v|8^wK{@qlj%+1b^?NI z$}l2tjp0g>K3O+p%yK<9!XqmQ?E9>z&(|^Pi~aSRwI5x$jaA62GFz9%fmO3t3a>cq zK8Xbv=5Ps~4mKN5+Eqw12(!PEyedFXv~VLxMB~HwT1Vfo51pQ#D8e$e4pFZ{&RC2P z5gTIzl{3!&(tor^BwZfR8j4k{7Rq#`riKXP2O-Bh66#WWK2w=z;iD9GLl+3 zpHIaI4#lQ&S-xBK8PiQ%dwOh?%BO~DCo06pN7<^dnZCN@NzY{_Z1>rrB0U|nC&+!2 z2y!oBcTd2;@lzyk(B=TkyZ)zy0deK05*Q0zk+o$@nun`VI1Er7pjq>8V zNmlW{p7S^Btgb(TA}jL(uR>`0w8gHP^T~Sh5Tkip^spk4SBAhC{TZU}_Z)UJw-}zm zPq{KBm!k)?P{`-(9?LFt&YN4s%SIZ-9lJ!Ws~B%exHOeVFk3~}HewnnH(d)qkLQ_d z6h>O)pEE{vbOVw}E+jdYC^wM+AAhaI(YAibUc@B#_mDss0Ji&BK{WG`4 zOk>vSNq(Bq2IB@s>>Rxm6Wv?h;ZXkpb1l8u|+_qXWdC*jjcPCixq;!%BVPSp#hP zqo`%cNf&YoQXHC$D=D45RiT|5ngPlh?0T~?lUf*O)){K@*Kbh?3RW1j9-T?%lDk@y z4+~?wKI%Y!-=O|_IuKz|=)F;V7ps=5@g)RrE;;tvM$gUhG>jHcw2Hr@fS+k^Zr~>G z^JvPrZc}_&d_kEsqAEMTMJw!!CBw)u&ZVzmq+ZworuaE&TT>$pYsd9|g9O^0orAe8 z221?Va!l1|Y5X1Y?{G7rt1sX#qFA^?RLG^VjoxPf63;AS=_mVDfGJKg73L zsGdnTUD40y(>S##2l|W2Cy!H(@@5KBa(#gs`vlz}Y~$ot5VsqPQ{{YtjYFvIumZzt zA{CcxZLJR|4#{j7k~Tu*jkwz8QA|5G1$Cl895R`Zyp;irp1{KN){kB30O8P1W5;@bG znvX74roeMmQlUi=v9Y%(wl$ZC#9tKNFpvi3!C}f1m6Ct|l2g%psc{TJp)@yu)*e2> z((p0Fg*8gJ!|3WZke9;Z{8}&NRkv7iP=#_y-F}x^y?2m%-D_aj^)f04%mneyjo_;) z6qc_Zu$q37d~X``*eP~Q>I2gg%rrV8v=kDfpp$=%Vj}hF)^dsSWygoN(A$g*E=Do6FX?&(@F#7pbiJ`;c0c@Ul zDqW_90Wm#5f2L<(Lf3)3TeXtI7nhYwRm(F;*r_G6K@OPW4H(Y3O5SjUzBC}u3d|eQ8*8d@?;zUPE+i#QNMn=r(ap?2SH@vo*m z3HJ%XuG_S6;QbWy-l%qU;8x;>z>4pMW7>R}J%QLf%@1BY(4f_1iixd-6GlO7Vp*yU zp{VU^3?s?90i=!#>H`lxT!q8rk>W_$2~kbpz7eV{3wR|8E=8**5?qn8#n`*(bt1xRQrdGxyx2y%B$qmw#>ZV$c7%cO#%JM1lY$Y0q?Yuo> ze9KdJoiM)RH*SB%^;TAdX-zEjA7@%y=!0=Zg%iWK7jVI9b&Dk}0$Af&08KHo+ zOwDhFvA(E|ER%a^cdh@^wLUlmIv6?_3=BvX8jKk92L=Y}7Jf5OGMfh` zBdR1wFCi-i5@`9km{isRb0O%TX+f~)KNaEz{rXQa89`YIF;EN&gN)cigu6mNh>?Cm zAO&Im2flv6D{jwm+y<%WsPe4!89n~KN|7}Cb{Z;XweER73r}Qp2 zz}WP4j}U0&(uD&9yGy6`!+_v-S(yG*iytsTR#x_Rc>=6u^vnRDnf1gP{#2>`ffrAC% zTZ5WQ@hAK;P;>kX{D)mIXe4%a5p=LO1xXH@8T?mz7Q@d)$3pL{{B!2{-v70L*o1AO+|n5beiw~ zk@(>m?T3{2k2c;NWc^`4@P&Z?BjxXJ@;x1qhn)9Mn*IFdt_J-dIqx5#d`NfyfX~m( zIS~5)MfZ2Uy?_4W`47i}u0ZgPh<{D|w_d#;D}Q&U$Q-G}xM1A@1f{#%A$jh6Qp&0hQ<0bPOM z-{1Wm&p%%#eb_?x7i;bol EfAhh=DF6Tf diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties deleted file mode 100644 index ffdc10e5..00000000 --- a/.mvn/wrapper/maven-wrapper.properties +++ /dev/null @@ -1,2 +0,0 @@ -distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.1/apache-maven-3.8.1-bin.zip -wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar diff --git a/build.gradle b/build.gradle index f7a584de..1dc7465e 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,7 @@ repositories { url = uri('https://repo.maven.apache.org/maven2') } - maven { + maven { url "https://fynarfin.jfrog.io/artifactory/fyn-libs-release-local" } diff --git a/src/.DS_Store b/src/.DS_Store deleted file mode 100644 index eeb17bb16445ed13d802efeb97e60656c2329620..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5T313w-vDmLGhG}w<1kN!GjR19=r(=J*a46iw0s=n$)7TlK0Se@Iib4 zA4g|)*W&M0ip;?5H#@B5Us{l??4L+K=14lj`iIO7k#I+Z~-8x7rwTbP%U4 z)oZG5+RF4ifffnifB@W zEir_x9E3^7ao@%l22DB${bZcSK3UibMcBeaxH=q!Z;)GNfEoDAz?kV)>HgpTz5f54 z#64z!8TeNW2xlwUYT}aY-TG~Dbl1A5r>G@HD%uMnXbY4)3)omqRowRSy3q=wT`MARiB4P|TvXl@W5XT2sZ?`Z=S z&#_yWNCK5*QO7Y(ix>!_HP?wPkvdk3t7fxv+C6E^N;Mb$g8^F|EXGQ z?K^kZweIN7{m#4O#LSX9t*6y6og8rQby}Zf-hN|Dv9!}{FCWhiyBjY}S*Lk9p4jAk zG)BnV{k$BR+0abOQDx^z+!JS(x%Yd>kLpgvL~np((6mwy%)&9Z?s_ zLRUl>5C(*SRb{{*g!0LZ_QR$$v-e_+TC zVC=DR2oFTLRG>>$zQj`+=^Kp42p zz?$oJ`Tl=(d;NboNqWM7Fz{b7pxQgh&JauTck8vq@m(82kDx3Z*En3Jz_3>_V)-h5 b2(*f@j-B0mC_25E$WzskUG{+DS& diff --git a/src/main/java/.DS_Store b/src/main/java/.DS_Store deleted file mode 100644 index 298f56fac9e03f44288d12a5363f1eeb691eac36..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK!AiqG5Z!I7ZYyFBg5oI`Z$+9G1rI{3_0XFT(SwR6wrC(`OPgA>R`MVEM}B~x zqcgkHf~YqUEHg0sCbKge=55%`2q9!+&~}9=3n3IJVy+0oH-hu1UnEDKJfOri^i>$^ z5T0;3mjA#2?ploE!B4uFX_cquX~11M_$tMt$nOi_-=dtNzm-q3X6LxO1vOyc6D;lXhO*O zX%ID3zp8psBi7@%rbCFF$khrfgTZ!Xy(r6@mC~>%2isfaqTJkA9}aV3X|=p_Sa09n z4eo~zk9ro2!*12GU~vRbV2sW2#_dL-imt&s${0lw5(C5lG4PieuxFZ>`^%G{Zzl$b zfp1{|&j$&L=vYh*>Z1c1T>=1R!L0>s{Bgt_Y0i}Y=RI$s?_$jm);_j6cx1xBsa8&-YOZvfuh!KEMj52k?k}pA(g)a z{)ivobKuSHrl<;)dZHEDk!Ig`c4k(7Ywdc7NDL;!K2e8=B$Tny!tfX2dDbOqIL``D zm>Od$NYfc5R4zq($9-gg_wE^uX@osXfPeO{#7-6z(}XhmbeQS0IMXqr`BT}0KL2F| z)~zbU({k)8{D{X{T2{T@PtjOuK6v;@%C>AjX@3YudKy+yH7N&C@t&WKqG}X7`;FGw zl^w-z(`0(mS$(bZDoXRDFv;mKLCEpDG#~2eKu_{vY3hV_LduqGopd&4v)!GouIg{^ z^yXbP+kM&Zs_hqB^Lb0IZT4Rso_@agGW$Be{AS*c1io$~PaQ7cJB(YfcODjbrt^=8 zUQ}9C@aP%v40s0af&q6H$;WrGxL?yV;2F5r4Dk7&pp1dT+M+o+Fqjqq*o4~%Z2m#y z9OW=@SX;ylMA%fIO%<-i5H=lt<>CT|wMCmw!Zjbltt?!JBFyTjuS__Jz@i^L1D=8F z3~=uU>G1u3aC84}g8bka@C^K)42b4_xIe(Vxx01gE%~lh&{HT2$JG|EQec>?7_od6 cuS1Q%uh;+v4r`0>K>Uw@p}`NHfq%-tZ=Kz1R{#J2 diff --git a/src/main/java/org/mifos/processor/bulk/OperationsAppConfig.java b/src/main/java/org/mifos/processor/bulk/OperationsAppConfig.java new file mode 100644 index 00000000..4ac5c5ac --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/OperationsAppConfig.java @@ -0,0 +1,23 @@ +package org.mifos.processor.bulk; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Component +public class OperationsAppConfig { + + @Value("${operations-app.contactpoint}") + public String operationAppContactPoint; + + @Value("${operations-app.endpoints.batch-transaction}") + public String batchTransactionEndpoint; + + public String batchTransactionUrl; + + @PostConstruct + private void setup() { + batchTransactionUrl = operationAppContactPoint + batchTransactionEndpoint; + } +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java new file mode 100644 index 00000000..30574a00 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java @@ -0,0 +1,57 @@ +package org.mifos.processor.bulk.camel.routes; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.zeebe.client.ZeebeClient; +import org.apache.camel.Exchange; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.model.RouteDefinition; +import org.mifos.processor.bulk.OperationsAppConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; + +public abstract class BaseRouteBuilder extends RouteBuilder { + + @Autowired + public ObjectMapper objectMapper; + + @Autowired + public OperationsAppConfig operationsAppConfig; + + @Autowired + ZeebeClient zeebeClient; + + public Logger logger = LoggerFactory.getLogger(this.getClass()); + + public RouteDefinition getBaseExternalApiRequestRouteDefinition(String routeId, HttpRequestMethod httpMethod) { + return from(String.format("direct:%s", routeId)) + .id(routeId) + .log("Starting external API request route: " + routeId) + .removeHeader("*") + .setHeader(Exchange.HTTP_METHOD, constant(httpMethod.text)) + .setHeader("X-Date", simple(ZonedDateTime.now( ZoneOffset.UTC ).format( DateTimeFormatter.ISO_INSTANT ))) + .setHeader("Content-Type", constant("application/json")); + } + + protected enum HttpRequestMethod { + GET("GET"), + POST("POST"), + PUT("PUT"), + DELETE("DELETE") + ; + + private final String text; + + HttpRequestMethod(String text) { + this.text = text; + } + + @Override + public String toString() { + return text; + } + } +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java index 7d1f6476..4a594f55 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java @@ -19,13 +19,7 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @Component -public class Routes extends RouteBuilder { - - @Autowired - ZeebeClient zeebeClient; - - @Value("${operations-app-config.host}") - String operationsAppHost; +public class Routes extends BaseRouteBuilder { @Value("${config.minimum-successful-tx-ratio}") double minimumSuccessfulTxRatio; @@ -42,7 +36,7 @@ private void routeCheckTransactions() { .id(id) .log("Fetching transaction details") //set request params - .to(operationsAppHost+ "/api/v1/batch/transactions") + .toD(operationsAppConfig.batchTransactionEndpoint) .process(exchange -> { // get response body JSONObject transfers = new JSONObject(exchange.getIn().getBody(String.class)); @@ -80,7 +74,7 @@ private void routeSampleTransactions() { .process(exchange -> { exchange.getIn().setHeader("batchId", exchange.getProperty(BATCH_ID)); }) - .toD(operationsAppHost + "/api/v1/batch/transactions") + .toD(operationsAppConfig.batchTransactionEndpoint) .process(exchange -> { // get response body diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index e6a4407b..f461b04d 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -27,6 +27,11 @@ zeebe: broker: contactpoint: "zeebe-zeebe-gateway:26500" +operations-app: + contactpoint: "https://ops-bk.ibank.financial" + endpoints: + batch-transaction: "/api/v1/batch/transactions" + cloud: aws: enabled: true @@ -59,8 +64,5 @@ bpmn: international-remittance-payer: "international_remittance_payer_process-{dfspid}" debit-party-process: "debit_party_process-{dfspid}" -operations-app-config: - host: "" - config: - minimum-successful-tx-ratio: 0.90 \ No newline at end of file + minimum-successful-tx-ratio: 0.90 diff --git a/src/test/java/org/mifos/pheeprocessorbulk/BulkProcessorApplicationTests.java b/src/test/java/org/mifos/processor/BulkProcessorApplicationTests.java similarity index 88% rename from src/test/java/org/mifos/pheeprocessorbulk/BulkProcessorApplicationTests.java rename to src/test/java/org/mifos/processor/BulkProcessorApplicationTests.java index 9c62bd7b..db5ed70a 100644 --- a/src/test/java/org/mifos/pheeprocessorbulk/BulkProcessorApplicationTests.java +++ b/src/test/java/org/mifos/processor/BulkProcessorApplicationTests.java @@ -1,4 +1,4 @@ -package org.mifos.pheeprocessorbulk; +package org.mifos.processor; import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; From cfaa9d27e483c5cfd77fcf5cb8ab83ffbb2e68dc Mon Sep 17 00:00:00 2001 From: Danish Jamal Date: Fri, 6 May 2022 06:22:02 +0530 Subject: [PATCH 014/156] Zeebe process started added --- .../camel/routes/ProcessorStartRoute.java | 50 +++++++++++++++++++ src/main/resources/application.yaml | 5 +- 2 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java new file mode 100644 index 00000000..fd5fa607 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -0,0 +1,50 @@ +package org.mifos.processor.bulk.camel.routes; + +import org.mifos.processor.bulk.zeebe.ZeebeProcessStarter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; + +@Component +public class ProcessorStartRoute extends BaseRouteBuilder{ + + @Autowired + private ZeebeProcessStarter zeebeProcessStarter; + + @Value("${bpmn.flows.bulk-processor}") + private String workflowId; + + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Override + public void configure() throws Exception { + setup(); + } + + private void setup() { + from("rest:POST:/bulk/transfer/{requestId}/{fileName}") + .process(exchange -> { + String fileName = exchange.getIn().getHeader("fileName", String.class); + String requestId = exchange.getIn().getHeader("requestId", String.class); + String batchId = UUID.randomUUID().toString(); + + logger.info("\n\n Filename: " + fileName + " \n\n"); + logger.info("\n\n BatchId: " + batchId + " \n\n"); + + Map variables = new HashMap<>(); + variables.put(BATCH_ID, batchId); + variables.put("fileName", fileName); + variables.put("requestId", requestId); + + zeebeProcessStarter.startZeebeWorkflow(workflowId, "", variables); + }); + } +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index f461b04d..ea862742 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -36,8 +36,8 @@ cloud: aws: enabled: true credentials: - access-key: ${AWS_ACCESS_KEY:AKIAX32JM37TYOG3QUJU} - secret-key: ${AWS_SECRET_KEY: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} + access-key: ${AWS_ACCESS_KEY:AKIAX32JM37TZOJ5AKFB} + secret-key: ${AWS_SECRET_KEY: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} region: static: us-east-2 stack: @@ -63,6 +63,7 @@ bpmn: international-remittance-payee: "international_remittance_payee_process-{dfspid}" international-remittance-payer: "international_remittance_payer_process-{dfspid}" debit-party-process: "debit_party_process-{dfspid}" + bulk-processor: "bulk_processor-ibank-usa" config: minimum-successful-tx-ratio: 0.90 From 21bc692ce33149dd291d8064b357dc8df9ba86da Mon Sep 17 00:00:00 2001 From: somanath21 <104554892+somanath21@users.noreply.github.com> Date: Thu, 30 Jun 2022 20:03:35 +0530 Subject: [PATCH 015/156] Create README.md --- README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 00000000..5e3ec501 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +#Auto-Trigger From a3630c9b3bf58a270984da83912efcb5eb3d1673 Mon Sep 17 00:00:00 2001 From: DanishJamal Date: Fri, 15 Jul 2022 08:14:44 +0530 Subject: [PATCH 016/156] Config properties --- src/main/resources/application.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index ea862742..85db0937 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -67,3 +67,7 @@ bpmn: config: minimum-successful-tx-ratio: 0.90 + ordering: true + formatting: true + mergeback: false + backpressure: false From 8a15f4b28bd6be1725cbed84ee60ca0806e213ed Mon Sep 17 00:00:00 2001 From: DanishJamal Date: Fri, 15 Jul 2022 08:14:44 +0530 Subject: [PATCH 017/156] PHEE-134 Implemented stub worker --- build.gradle | 2 + .../org/mifos/processor/bulk/HealthCheck.java | 13 +- .../camel/routes/ProcessorStartRoute.java | 36 ++++++ .../processor/bulk/camel/routes/RouteId.java | 22 ++++ .../bulk/file/AwsFileTransferImpl.java | 11 +- .../bulk/file/AzureFileTransferImpl.java | 13 ++ .../bulk/file/FileTransferService.java | 4 + .../mifos/processor/bulk/kafka/Consumers.java | 120 ------------------ .../kafka/config/KafkaConsumerConfig.java | 40 ------ .../kafka/config/KafkaProducerConfig.java | 35 ----- .../bulk/kafka/config/KafkaTopicConfig.java | 40 ------ .../processor/bulk/zeebe/ZeebeVariables.java | 9 +- .../processor/bulk/zeebe/ZeebeWorkers.java | 18 +-- .../bulk/zeebe/worker/ApprovalWorker.java | 24 ++++ .../bulk/zeebe/worker/BaseWorker.java | 60 +++++++++ .../bulk/zeebe/worker/BatchStatusWorker.java | 12 ++ .../bulk/zeebe/worker/FormattingWorker.java | 21 +++ .../bulk/zeebe/worker/MergeBackWorker.java | 21 +++ .../bulk/zeebe/worker/OrderingWorker.java | 21 +++ .../bulk/zeebe/worker/PartyLookupWorker.java | 26 ++++ .../bulk/zeebe/worker/SplittingWorker.java | 22 ++++ .../processor/bulk/zeebe/worker/Worker.java | 23 ++++ .../bulk/zeebe/worker/WorkerConfig.java | 28 ++++ src/main/resources/application.yaml | 15 +-- 24 files changed, 367 insertions(+), 269 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java delete mode 100644 src/main/java/org/mifos/processor/bulk/kafka/Consumers.java delete mode 100644 src/main/java/org/mifos/processor/bulk/kafka/config/KafkaConsumerConfig.java delete mode 100644 src/main/java/org/mifos/processor/bulk/kafka/config/KafkaProducerConfig.java delete mode 100644 src/main/java/org/mifos/processor/bulk/kafka/config/KafkaTopicConfig.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/ApprovalWorker.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/BaseWorker.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/FormattingWorker.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/PartyLookupWorker.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java diff --git a/build.gradle b/build.gradle index 1dc7465e..ae240d5f 100644 --- a/build.gradle +++ b/build.gradle @@ -38,6 +38,8 @@ dependencies { implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.12.0' implementation 'org.springframework.kafka:spring-kafka:2.5.8.RELEASE' testImplementation 'org.springframework.boot:spring-boot-starter-test:2.5.2' + + implementation 'org.apache.camel.springboot:camel-mail-starter:3.4.0' } group = 'org.mifos' diff --git a/src/main/java/org/mifos/processor/bulk/HealthCheck.java b/src/main/java/org/mifos/processor/bulk/HealthCheck.java index 9ecbe2a4..f2ee6569 100644 --- a/src/main/java/org/mifos/processor/bulk/HealthCheck.java +++ b/src/main/java/org/mifos/processor/bulk/HealthCheck.java @@ -20,6 +20,7 @@ import java.util.UUID; @Component +@Deprecated public class HealthCheck extends RouteBuilder { private Logger logger = LoggerFactory.getLogger(this.getClass()); @@ -43,12 +44,6 @@ public class HealthCheck extends RouteBuilder { @Value("${zeebe.client.evenly-allocated-max-jobs}") private int workerMaxJobs; - @Value(value = "${kafka.topic.gsma.name}") - private String gsmaTopicName; - - @Value(value = "${kafka.topic.slcb.name}") - private String slcbTopicName; - @Override public void configure() { from("rest:GET:/") @@ -68,7 +63,7 @@ public void configure() { CsvSchema schema = CsvSchema.emptySchema().withHeader(); MappingIterator readValues = csvMapper.readerWithSchemaFor(Transaction.class).with(schema).readValues(csvFile); - while (readValues.hasNext()) { + /*while (readValues.hasNext()) { Transaction current = readValues.next(); current.setBatchId(batchId); System.out.println(objectMapper.writeValueAsString(current)); @@ -76,9 +71,9 @@ public void configure() { kafkaTemplate.send(gsmaTopicName, objectMapper.writeValueAsString(current)); else if (current.getPayment_mode().equals("sclb")) kafkaTemplate.send(slcbTopicName, objectMapper.writeValueAsString(current)); - } + }*/ }) .setHeader(Exchange.HTTP_RESPONSE_CODE, constant(200)) .setBody(constant("")); } -} \ No newline at end of file +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index fd5fa607..90fe0087 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -1,12 +1,22 @@ package org.mifos.processor.bulk.camel.routes; +import io.undertow.util.MultipartParser; +import org.apache.camel.attachment.Attachment; +import org.apache.camel.attachment.AttachmentMessage; +import org.mifos.processor.bulk.file.FileTransferService; import org.mifos.processor.bulk.zeebe.ZeebeProcessStarter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.client.MultipartBodyBuilder; import org.springframework.stereotype.Component; +import org.springframework.web.multipart.MultipartFile; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.UUID; @@ -19,6 +29,13 @@ public class ProcessorStartRoute extends BaseRouteBuilder{ @Autowired private ZeebeProcessStarter zeebeProcessStarter; + @Autowired + @Qualifier("awsStorage") + private FileTransferService fileTransferService; + + @Value("${application.bucket-name}") + private String bucketName; + @Value("${bpmn.flows.bulk-processor}") private String workflowId; @@ -31,6 +48,7 @@ public void configure() throws Exception { private void setup() { from("rest:POST:/bulk/transfer/{requestId}/{fileName}") + .unmarshal().mimeMultipart("multipart/*") .process(exchange -> { String fileName = exchange.getIn().getHeader("fileName", String.class); String requestId = exchange.getIn().getHeader("requestId", String.class); @@ -38,6 +56,24 @@ private void setup() { logger.info("\n\n Filename: " + fileName + " \n\n"); logger.info("\n\n BatchId: " + batchId + " \n\n"); + + File file = new File(fileName); + file.setWritable(true); + file.setReadable(true); + + String csvData = exchange.getIn().getBody(String.class); + FileWriter fileWriter = new FileWriter(file); + fileWriter.write(csvData); + fileWriter.close(); + + logger.info(csvData); + logger.info(""+file.length()); + logger.info(file.getAbsolutePath()); + + String nm = fileTransferService.uploadFile(file, bucketName); + + logger.info("File uploaded {}", nm); + Map variables = new HashMap<>(); variables.put(BATCH_ID, batchId); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java b/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java new file mode 100644 index 00000000..051ca08f --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java @@ -0,0 +1,22 @@ +package org.mifos.processor.bulk.camel.routes; + +public enum RouteId { + + PARTY_LOOKUP("direct:partyLookup"), + APPROVAL("direct:approval"), + ORDERING("direct:ordering"), + SPLITTING("direct:splitting"), + FORMATTING("direct:formatting"), + BATCH_STATUS("direct:batchStatus"), + MERGE_BACK("direct:mergeSubBatch"); + + private final String value; + + private RouteId(String s) { + value = s; + } + + public String getValue() { + return value; + } +} diff --git a/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java b/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java index 80895b02..05b14b27 100644 --- a/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java +++ b/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java @@ -28,9 +28,14 @@ public class AwsFileTransferImpl implements FileTransferService { public String uploadFile(MultipartFile file, String bucketName) { File fileObj = convertMultiPartFileToFile(file); - String fileName = System.currentTimeMillis() + "_" + file.getOriginalFilename(); - s3Client.putObject(new PutObjectRequest(bucketName, fileName, fileObj)); - fileObj.delete(); + return uploadFile(fileObj, bucketName); + } + + @Override + public String uploadFile(File file, String bucketName) { + String fileName = System.currentTimeMillis() + "_" + file.getName(); + s3Client.putObject(new PutObjectRequest(bucketName, fileName, file)); + file.delete(); return fileName; } diff --git a/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java b/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java index e38b721a..ffa93d8a 100644 --- a/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java +++ b/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java @@ -11,6 +11,7 @@ import org.springframework.web.multipart.MultipartFile; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; @@ -40,6 +41,18 @@ public String uploadFile(MultipartFile file, String bucketName) { return null; } + @Override + public String uploadFile(File file, String bucketName) { + try { + String fileName = System.currentTimeMillis() + "_" + file.getName(); + client.containerName(bucketName).blobName(fileName).buildClient().upload(Files.newInputStream(file.toPath()), file.length()); + return fileName; + } catch (IOException e) { + logger.error("Error uploading file to Azure", e); + } + return null; + } + @Override public byte[] downloadFile(String fileName, String bucketName) { try { diff --git a/src/main/java/org/mifos/processor/bulk/file/FileTransferService.java b/src/main/java/org/mifos/processor/bulk/file/FileTransferService.java index a139a4b1..a9789087 100644 --- a/src/main/java/org/mifos/processor/bulk/file/FileTransferService.java +++ b/src/main/java/org/mifos/processor/bulk/file/FileTransferService.java @@ -3,11 +3,15 @@ import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; +import java.io.File; + @Service public interface FileTransferService { String uploadFile(MultipartFile file, String bucketName); + String uploadFile(File file, String bucketName); + byte[] downloadFile(String fileName, String bucketName); void deleteFile(String fileName, String bucketName); diff --git a/src/main/java/org/mifos/processor/bulk/kafka/Consumers.java b/src/main/java/org/mifos/processor/bulk/kafka/Consumers.java deleted file mode 100644 index 70317a10..00000000 --- a/src/main/java/org/mifos/processor/bulk/kafka/Consumers.java +++ /dev/null @@ -1,120 +0,0 @@ -package org.mifos.processor.bulk.kafka; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.mifos.connector.common.channel.dto.TransactionChannelRequestDTO; -import org.mifos.connector.common.gsma.dto.GSMATransaction; -import org.mifos.connector.common.gsma.dto.GsmaParty; -import org.mifos.connector.common.mojaloop.dto.MoneyData; -import org.mifos.connector.common.mojaloop.dto.Party; -import org.mifos.connector.common.mojaloop.dto.PartyIdInfo; -import org.mifos.connector.common.mojaloop.dto.TransactionType; -import org.mifos.connector.common.mojaloop.type.IdentifierType; -import org.mifos.processor.bulk.schema.Transaction; -import org.mifos.processor.bulk.zeebe.ZeebeProcessStarter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.kafka.annotation.KafkaListener; -import org.springframework.stereotype.Service; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.mifos.connector.common.mojaloop.type.InitiatorType.CONSUMER; -import static org.mifos.connector.common.mojaloop.type.Scenario.TRANSFER; -import static org.mifos.connector.common.mojaloop.type.TransactionRole.PAYER; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.GSMA_CHANNEL_REQUEST; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INITIATOR_FSPID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.IS_RTP_REQUEST; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_ID_TYPE; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_FSPID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TRANSACTION_TYPE; - -@Service -public class Consumers { - - @Value("#{'${dfspids}'.split(',')}") - private List dfspIds; - - @Value("${bpmn.flows.international-remittance-payer}") - private String internationalRemittancePayer; - - @Autowired - private ObjectMapper objectMapper; - - @Autowired - private ZeebeProcessStarter zeebeProcessStarter; - - @KafkaListener(topics = "${kafka.topic.gsma.name}", groupId = "group_id") - public void listenTopicGsma(String message) throws JsonProcessingException { - System.out.println("Received Message in topic GSMA and group group_id: " + message); - - Transaction transaction = objectMapper.readValue((String) message, Transaction.class); - String tenantId = "ibank-usa"; - - GSMATransaction gsmaChannelRequest = new GSMATransaction(); - gsmaChannelRequest.setAmount(transaction.getAmount()); - gsmaChannelRequest.setCurrency(transaction.getCurrency()); - gsmaChannelRequest.setRequestingLei("ibank-usa"); - gsmaChannelRequest.setReceivingLei("ibank-india"); - GsmaParty creditParty = new GsmaParty(); - creditParty.setKey("msisdn"); - creditParty.setValue(transaction.getAccount_number()); - GsmaParty debitParty = new GsmaParty(); - debitParty.setKey("msisdn"); - debitParty.setValue(transaction.getAccount_number()); - gsmaChannelRequest.setCreditParty(new GsmaParty[]{creditParty}); - gsmaChannelRequest.setDebitParty(new GsmaParty[]{debitParty}); -// gsmaChannelRequest.setInternationalTransferInformation().setReceivingAmount(gsmaChannelRequest.getAmount()); - - TransactionChannelRequestDTO channelRequest = new TransactionChannelRequestDTO(); // Fineract Object - Party payee = new Party(new PartyIdInfo(IdentifierType.MSISDN, transaction.getAccount_number())); - Party payer = new Party(new PartyIdInfo(IdentifierType.MSISDN, "7543010")); - - MoneyData moneyData = new MoneyData(); - moneyData.setAmount(transaction.getAmount()); - moneyData.setCurrency(transaction.getCurrency()); - - channelRequest.setPayer(payer); - channelRequest.setPayee(payee); - channelRequest.setAmount(moneyData); - - TransactionType transactionType = new TransactionType(); - transactionType.setInitiator(PAYER); - transactionType.setInitiatorType(CONSUMER); - transactionType.setScenario(TRANSFER); - - Map extraVariables = new HashMap<>(); - extraVariables.put(IS_RTP_REQUEST, false); - extraVariables.put(TRANSACTION_TYPE, "inttransfer"); - extraVariables.put(TENANT_ID, tenantId); - - extraVariables.put(BATCH_ID, transaction.getBatchId()); - - String tenantSpecificBpmn = internationalRemittancePayer.replace("{dfspid}", tenantId); - channelRequest.setTransactionType(transactionType); - - PartyIdInfo requestedParty = (boolean)extraVariables.get(IS_RTP_REQUEST) ? channelRequest.getPayer().getPartyIdInfo() : channelRequest.getPayee().getPartyIdInfo(); - extraVariables.put(PARTY_ID_TYPE, requestedParty.getPartyIdType()); - extraVariables.put(PARTY_ID, requestedParty.getPartyIdentifier()); - - extraVariables.put(GSMA_CHANNEL_REQUEST, objectMapper.writeValueAsString(gsmaChannelRequest)); - extraVariables.put(PARTY_LOOKUP_FSPID, gsmaChannelRequest.getReceivingLei()); - extraVariables.put(INITIATOR_FSPID, gsmaChannelRequest.getRequestingLei()); - - String transactionId = zeebeProcessStarter.startZeebeWorkflow(tenantSpecificBpmn, - objectMapper.writeValueAsString(channelRequest), - extraVariables); - - System.out.println("GSMA Transaction Started with: " + transactionId); - } - - @KafkaListener(topics = "${kafka.topic.slcb.name}", groupId = "group_id") - public void listenTopicSlcb(String message) { - System.out.println("Received Message in topic SLCB and group group_id: " + message); - } -} diff --git a/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaConsumerConfig.java b/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaConsumerConfig.java deleted file mode 100644 index 5b689a39..00000000 --- a/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaConsumerConfig.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.mifos.processor.bulk.kafka.config; - -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.kafka.annotation.EnableKafka; -import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; -import org.springframework.kafka.core.ConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; - -import java.util.HashMap; -import java.util.Map; - -@EnableKafka -@Configuration -public class KafkaConsumerConfig { - - @Value(value = "${kafka.bootstrapAddress}") - private String bootstrapAddress; - - @Bean - public ConsumerFactory consumerFactory() { - Map props = new HashMap<>(); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapAddress); - props.put(ConsumerConfig.GROUP_ID_CONFIG, "group_id"); - props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - return new DefaultKafkaConsumerFactory<>(props); - } - - @Bean - public ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory() { - ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); - factory.setConsumerFactory(consumerFactory()); - return factory; - } - -} diff --git a/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaProducerConfig.java b/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaProducerConfig.java deleted file mode 100644 index 4ef5d8d9..00000000 --- a/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaProducerConfig.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.mifos.processor.bulk.kafka.config; - -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.serialization.StringSerializer; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.core.ProducerFactory; - -import java.util.HashMap; -import java.util.Map; - -@Configuration -public class KafkaProducerConfig { - - @Value(value = "${kafka.bootstrapAddress}") - private String bootstrapAddress; - - @Bean - public ProducerFactory producerFactory() { - Map configProps = new HashMap<>(); - configProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapAddress); - configProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); - configProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); - return new DefaultKafkaProducerFactory<>(configProps); - } - - @Bean - public KafkaTemplate kafkaTemplate() { - return new KafkaTemplate<>(producerFactory()); - } - -} diff --git a/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaTopicConfig.java b/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaTopicConfig.java deleted file mode 100644 index 625ddee2..00000000 --- a/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaTopicConfig.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.mifos.processor.bulk.kafka.config; - -import org.apache.kafka.clients.admin.AdminClientConfig; -import org.apache.kafka.clients.admin.NewTopic; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.kafka.core.KafkaAdmin; - -import java.util.HashMap; -import java.util.Map; - -@Configuration -public class KafkaTopicConfig { - @Value(value = "${kafka.bootstrapAddress}") - private String bootstrapAddress; - - @Value(value = "${kafka.topic.gsma.name}") - private String gsmaTopicName; - - @Value(value = "${kafka.topic.slcb.name}") - private String slcbTopicName; - - @Bean - public KafkaAdmin kafkaAdmin() { - Map configs = new HashMap<>(); - configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapAddress); - return new KafkaAdmin(configs); - } - - @Bean - public NewTopic gsmaTopic() { - return new NewTopic(gsmaTopicName, 1, (short) 1); - } - - @Bean - public NewTopic slcbTopic() { - return new NewTopic(slcbTopicName, 1, (short) 1); - } -} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 47a18a67..b031db71 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -24,4 +24,11 @@ private ZeebeVariables() { public static final String BATCH_ID = "batchId"; public static final String IS_SAMPLE_READY = "isSampleReady"; public static final String SAMPLED_TX_IDS = "sampledTransactionIds"; -} \ No newline at end of file + + public static final String PARTY_LOOKUP_FAILED = "partyLookupFailed"; + public static final String APPROVAL_FAILED = "approvalFailed"; + public static final String ORDERING_FAILED = "orderingFailed"; + public static final String SPLITTING_FAILED = "splittingFailed"; + public static final String FORMATTING_FAILED = "formattingFailed"; + public static final String MERGE_FAILED = "mergeFailed"; +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java index 95840810..1c1d1c21 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java @@ -28,6 +28,7 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @Component +@Deprecated public class ZeebeWorkers { private Logger logger = LoggerFactory.getLogger(this.getClass()); @@ -44,28 +45,19 @@ public class ZeebeWorkers { @Qualifier("awsStorage") private FileTransferService fileTransferService; - @Autowired - private KafkaTemplate kafkaTemplate; - @Value("${application.bucket-name}") private String bucketName; @Value("${zeebe.client.evenly-allocated-max-jobs}") private int workerMaxJobs; - @Value(value = "${kafka.topic.gsma.name}") - private String gsmaTopicName; - - @Value(value = "${kafka.topic.slcb.name}") - private String slcbTopicName; - @Autowired private CamelContext camelContext; @Autowired private ProducerTemplate producerTemplate; - @PostConstruct + //@PostConstruct public void setupWorkers() { workerBulkProcessor(); workerCheckTransactions(); @@ -88,14 +80,14 @@ private void workerBulkProcessor(){ CsvSchema schema = CsvSchema.emptySchema().withHeader(); MappingIterator readValues = csvMapper.readerWithSchemaFor(Transaction.class).with(schema).readValues(csvFile); - while (readValues.hasNext()) { + /*while (readValues.hasNext()) { Transaction current = readValues.next(); current.setBatchId(batchId); if (current.getPayment_mode().equals("gsma")) kafkaTemplate.send(gsmaTopicName, objectMapper.writeValueAsString(current)); else if (current.getPayment_mode().equals("sclb")) kafkaTemplate.send(slcbTopicName, objectMapper.writeValueAsString(current)); - } + }*/ client.newCompleteCommand(job.getKey()) .send(); @@ -149,4 +141,4 @@ private void workerSampleTransactions(){ .maxJobsActive(workerMaxJobs) .open(); } -} \ No newline at end of file +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/ApprovalWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/ApprovalWorker.java new file mode 100644 index 00000000..8171904b --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/ApprovalWorker.java @@ -0,0 +1,24 @@ +package org.mifos.processor.bulk.zeebe.worker; + +import org.springframework.stereotype.Component; + +import java.util.Map; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.APPROVAL_FAILED; + +@Component +public class ApprovalWorker extends BaseWorker { + + @Override + public void setup() { + newWorker(Worker.APPROVAL, (client, job) -> { + Map variables = job.getVariablesAsMap(); + + if (workerConfig.isApprovalWorkerEnabled) { + variables.put(APPROVAL_FAILED, false); + } + + client.newCompleteCommand(job.getKey()).variables(variables).send(); + }); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BaseWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BaseWorker.java new file mode 100644 index 00000000..fe0d7abe --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BaseWorker.java @@ -0,0 +1,60 @@ +package org.mifos.processor.bulk.zeebe.worker; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import io.camunda.zeebe.client.ZeebeClient; +import io.camunda.zeebe.client.api.worker.JobHandler; +import org.apache.camel.CamelContext; +import org.apache.camel.Exchange; +import org.apache.camel.ProducerTemplate; +import org.mifos.processor.bulk.camel.routes.RouteId; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Component +public abstract class BaseWorker { + + @Autowired + private ZeebeClient zeebeClient; + + @Autowired + private CsvMapper csvMapper; + + @Autowired + private ObjectMapper objectMapper; + + @Value("${application.bucket-name}") + private String bucketName; + + @Value("${zeebe.client.evenly-allocated-max-jobs}") + private int workerMaxJobs; + + @Autowired + private CamelContext camelContext; + + @Autowired + private ProducerTemplate producerTemplate; + + @Autowired + protected WorkerConfig workerConfig; + + @PostConstruct + public abstract void setup(); + + public void newWorker(Worker worker, JobHandler handler) { + zeebeClient.newWorker() + .jobType(worker.getValue()) + .handler(handler) + .name(worker.getValue()) + .maxJobsActive(workerMaxJobs) + .open(); + } + + public void sendToCamelRoute(RouteId routeId, Exchange exchange) { + producerTemplate.send(routeId.getValue(), exchange); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java new file mode 100644 index 00000000..e3113f16 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java @@ -0,0 +1,12 @@ +package org.mifos.processor.bulk.zeebe.worker; + +public class BatchStatusWorker extends BaseWorker { + + @Override + public void setup() { + newWorker(Worker.BATCH_STATUS, (client, job) -> { + client.newCompleteCommand(job.getKey()).send(); + }); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/FormattingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/FormattingWorker.java new file mode 100644 index 00000000..53905404 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/FormattingWorker.java @@ -0,0 +1,21 @@ +package org.mifos.processor.bulk.zeebe.worker; + +import java.util.Map; + +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FORMATTING_FAILED; + +public class FormattingWorker extends BaseWorker { + + @Override + public void setup() { + newWorker(Worker.FORMATTING, (client, job) -> { + Map variables = job.getVariablesAsMap(); + if (workerConfig.isFormattingWorkerEnabled) { + variables.put(FORMATTING_FAILED, false); + } + + client.newCompleteCommand(job.getKey()).variables(variables).send(); + }); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java new file mode 100644 index 00000000..8f4a0fe8 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java @@ -0,0 +1,21 @@ +package org.mifos.processor.bulk.zeebe.worker; + +import java.util.Map; + +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_FAILED; + +public class MergeBackWorker extends BaseWorker { + + @Override + public void setup() { + newWorker(Worker.MERGE_BACK, (client, job) -> { + Map variables = job.getVariablesAsMap(); + if (workerConfig.isMergeBackWorkerEnabled) { + variables.put(MERGE_FAILED, false); + } + + client.newCompleteCommand(job.getKey()).variables(variables).send(); + }); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java new file mode 100644 index 00000000..899ef522 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java @@ -0,0 +1,21 @@ +package org.mifos.processor.bulk.zeebe.worker; + +import java.util.Map; + +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERING_FAILED; + +public class OrderingWorker extends BaseWorker { + + @Override + public void setup() { + newWorker(Worker.ORDERING, (client, job) -> { + Map variables = job.getVariablesAsMap(); + if (workerConfig.isOrderingWorkerEnabled) { + variables.put(ORDERING_FAILED, false); + } + + client.newCompleteCommand(job.getKey()).variables(variables).send(); + }); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/PartyLookupWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/PartyLookupWorker.java new file mode 100644 index 00000000..e8d4b8a5 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/PartyLookupWorker.java @@ -0,0 +1,26 @@ +package org.mifos.processor.bulk.zeebe.worker; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.Map; + +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_FAILED; + +@Component +public class PartyLookupWorker extends BaseWorker { + + @Override + public void setup() { + newWorker(Worker.PARTY_LOOKUP, (client, job) -> { + Map variables = job.getVariablesAsMap(); + + if (workerConfig.isPartyLookUpWorkerEnabled) { + variables.put(PARTY_LOOKUP_FAILED, false); + } + + client.newCompleteCommand(job.getKey()).variables(variables).send(); + }); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java new file mode 100644 index 00000000..f16c02ae --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java @@ -0,0 +1,22 @@ +package org.mifos.processor.bulk.zeebe.worker; + +import java.util.Map; + +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_FAILED; + +public class SplittingWorker extends BaseWorker { + + @Override + public void setup() { + newWorker(Worker.SPLITTING, (client, job) -> { + Map variables = job.getVariablesAsMap(); + if (workerConfig.isSplittingWorkerEnabled) { + variables.put(SPLITTING_FAILED, false); + } + + client.newCompleteCommand(job.getKey()).variables(variables).send(); + }); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java new file mode 100644 index 00000000..df6fd2c3 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java @@ -0,0 +1,23 @@ +package org.mifos.processor.bulk.zeebe.worker; + +public enum Worker { + + PARTY_LOOKUP("partyLookup"), + APPROVAL("approval"), + ORDERING("ordering"), + SPLITTING("splitting"), + FORMATTING("formatting"), + BATCH_STATUS("batchStatus"), + MERGE_BACK("mergeSubBatch"); + + private final String value; + + private Worker(String s) { + value = s; + } + + public String getValue() { + return value; + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java new file mode 100644 index 00000000..1c0235fb --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java @@ -0,0 +1,28 @@ +package org.mifos.processor.bulk.zeebe.worker; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class WorkerConfig { + + @Value("${config.partylookup}") + public boolean isPartyLookUpWorkerEnabled; + + @Value("${config.approval}") + public boolean isApprovalWorkerEnabled; + + @Value("${config.ordering}") + public boolean isOrderingWorkerEnabled; + + @Value("${config.splitting}") + public boolean isSplittingWorkerEnabled; + + @Value("${config.formatting}") + public boolean isFormattingWorkerEnabled; + + @Value("${config.mergeback}") + public boolean isMergeBackWorkerEnabled; + + +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index ea862742..2f7be99c 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -9,14 +9,6 @@ camel: application: bucket-name: paymenthub-ee-dev -kafka: - bootstrapAddress: "kafka:9092" - topic: - gsma: - name: gsma - slcb: - name: slcb - zeebe: client: max-execution-threads: 1000 @@ -67,3 +59,10 @@ bpmn: config: minimum-successful-tx-ratio: 0.90 + partylookup: false + approval: false + ordering: true + splitting: false + formatting: true + mergeback: false + backpressure: false From 9fb6014c86eaeeb8d2c4a535b2506efc64845587 Mon Sep 17 00:00:00 2001 From: DanishJamal Date: Wed, 3 Aug 2022 16:30:39 +0530 Subject: [PATCH 018/156] Added route for downloading file from cloud --- .../bulk/camel/config/CamelProperties.java | 4 ++ .../camel/routes/FileDownloadingRoute.java | 41 +++++++++++++++++++ .../bulk/zeebe/worker/WorkerConfig.java | 12 +++--- src/main/resources/application.yaml | 22 ++++++---- 4 files changed, 66 insertions(+), 13 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/FileDownloadingRoute.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 0f3ffb1e..665af581 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -7,4 +7,8 @@ private CamelProperties() {} public static final String AUTH_TYPE = "authType"; public static final String IS_BATCH_READY = "isBatchReady"; // camel property to check if batch is ready for sampling + public static final String SERVER_FILE_NAME = "serverFileNam"; + + public static final String LOCAL_FILE_PATH = "localFilePath"; + } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileDownloadingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileDownloadingRoute.java new file mode 100644 index 00000000..353545ad --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileDownloadingRoute.java @@ -0,0 +1,41 @@ +package org.mifos.processor.bulk.camel.routes; + +import org.mifos.processor.bulk.file.FileTransferService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.io.File; +import java.io.FileOutputStream; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; + +@Component +public class FileDownloadingRoute extends BaseRouteBuilder{ + + @Autowired + @Qualifier("awsStorage") + private FileTransferService fileTransferService; + + @Value("${application.bucket-name}") + private String bucketName; + + @Override + public void configure() throws Exception { + from("direct:download-file") + .id("download-file") + .log("Started download-file route") + .process(exchange -> { + String filename = exchange.getProperty(SERVER_FILE_NAME, String.class); + + byte[] csvFile = fileTransferService.downloadFile(filename, bucketName); + File file = new File(filename); + try (FileOutputStream fos = new FileOutputStream(file)) { + fos.write(csvFile); + } + exchange.setProperty(LOCAL_FILE_PATH, file.getAbsolutePath()); + }); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java index 1c0235fb..0eab8ecd 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java @@ -6,22 +6,22 @@ @Component public class WorkerConfig { - @Value("${config.partylookup}") + @Value("${config.partylookup.enable}") public boolean isPartyLookUpWorkerEnabled; - @Value("${config.approval}") + @Value("${config.approval.enable}") public boolean isApprovalWorkerEnabled; - @Value("${config.ordering}") + @Value("${config.ordering.enable}") public boolean isOrderingWorkerEnabled; - @Value("${config.splitting}") + @Value("${config.splitting.enable}") public boolean isSplittingWorkerEnabled; - @Value("${config.formatting}") + @Value("${config.formatting.enable}") public boolean isFormattingWorkerEnabled; - @Value("${config.mergeback}") + @Value("${config.mergeback.enable}") public boolean isMergeBackWorkerEnabled; diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 2f7be99c..3a9bd630 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -59,10 +59,18 @@ bpmn: config: minimum-successful-tx-ratio: 0.90 - partylookup: false - approval: false - ordering: true - splitting: false - formatting: true - mergeback: false - backpressure: false + partylookup: + enable: false + approval: + enable: false + ordering: + enable: true + splitting: + enable: false + batch-size: 10 + formatting: + enable: true + mergeback: + enable: false + backpressure: + enable: false From 32a594b2e8fb72666dfc2d0b7613be7c1ddef6f0 Mon Sep 17 00:00:00 2001 From: DanishJamal Date: Thu, 4 Aug 2022 14:05:31 +0530 Subject: [PATCH 019/156] Implemented splitting worker --- .../bulk/camel/config/CamelProperties.java | 8 ++ ...leDownloadingRoute.java => FileRoute.java} | 25 ++++- .../bulk/camel/routes/SplittingRoute.java | 103 ++++++++++++++++++ .../processor/bulk/zeebe/ZeebeVariables.java | 6 + .../bulk/zeebe/worker/BaseWorker.java | 2 +- .../bulk/zeebe/worker/SplittingWorker.java | 40 ++++++- src/main/resources/application.yaml | 2 +- 7 files changed, 180 insertions(+), 6 deletions(-) rename src/main/java/org/mifos/processor/bulk/camel/routes/{FileDownloadingRoute.java => FileRoute.java} (55%) create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 665af581..f1f85017 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -11,4 +11,12 @@ private CamelProperties() {} public static final String LOCAL_FILE_PATH = "localFilePath"; + public static final String SUB_BATCH_FILE_ARRAY = "subBatchFileArray"; + + public static final String SUB_BATCH_COUNT = "subBatchCount"; + + public static final String SUB_BATCH_CREATED = "subBatchCreated"; + + public static final String SERVER_SUB_BATCH_FILE_NAME_ARRAY = "serverSubBatchFileName"; + } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileDownloadingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java similarity index 55% rename from src/main/java/org/mifos/processor/bulk/camel/routes/FileDownloadingRoute.java rename to src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java index 353545ad..a204c06c 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FileDownloadingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java @@ -13,7 +13,7 @@ import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; @Component -public class FileDownloadingRoute extends BaseRouteBuilder{ +public class FileRoute extends BaseRouteBuilder { @Autowired @Qualifier("awsStorage") @@ -24,8 +24,14 @@ public class FileDownloadingRoute extends BaseRouteBuilder{ @Override public void configure() throws Exception { + + /** + * Downloads the file from cloud, stores in local and returns the file path + * Input the file name through exchange variable: [SERVER_FILE_NAME] + * Output the local file path through exchange variable: [LOCAL_FILE_PATH] + */ from("direct:download-file") - .id("download-file") + .id("direct:download-file") .log("Started download-file route") .process(exchange -> { String filename = exchange.getProperty(SERVER_FILE_NAME, String.class); @@ -37,5 +43,20 @@ public void configure() throws Exception { } exchange.setProperty(LOCAL_FILE_PATH, file.getAbsolutePath()); }); + + /** + * Uploads the file to cloud and returns the file name in cloud + * Input the local file path through exchange variable: [LOCAL_FILE_PATH] + * Output the server file name through exchange variable: [SERVER_FILE_NAME] + */ + from("direct:upload-file") + .id("direct:upload-file") + .log("Uploading file") + .process(exchange -> { + String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); + String serverFileName = fileTransferService.uploadFile(new File(filepath), bucketName); + exchange.setProperty(SERVER_FILE_NAME, serverFileName); + logger.info("Uploaded file: {}", serverFileName); + }); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java new file mode 100644 index 00000000..4dd6b691 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java @@ -0,0 +1,103 @@ +package org.mifos.processor.bulk.camel.routes; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.FileWriter; +import java.util.ArrayList; +import java.util.List; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_FAILED; + +@Component +public class SplittingRoute extends BaseRouteBuilder { + + @Value("${config.splitting.sub-batch-size}") + private int subBatchSize; + + @Override + public void configure() throws Exception { + + /** + * Base route for starting the splitting process. Refer below routes for more info + * 1. direct:create-sub-batch-file + * 2. direct:upload-sub-batch-file + */ + from(RouteId.SPLITTING.getValue()) + .id(RouteId.SPLITTING.getValue()) + .log("Starting route " + RouteId.SPLITTING.name()) + .process(exchange -> exchange.setProperty(LOCAL_FILE_PATH, "1659424507931_ph-ee-bulk-demo-5.csv")) + .to("direct:create-sub-batch-file") + .choice() + .when(exchange -> exchange.getProperty(SUB_BATCH_CREATED, Boolean.class)) + .to("direct:upload-sub-batch-file") + .otherwise() + .log("No sub batch created, so skipping upload") + .end() + .process(exchange -> exchange.setProperty(SPLITTING_FAILED, false)); + + /** + * Creates the sub-batch CSVs + */ + from("direct:create-sub-batch-file") + .id("direct:create-sub-batch-file") + .log("Creating sub-batch file") + .process(exchange -> { + String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); + BufferedReader reader = new BufferedReader(new FileReader(filepath)); + String header = reader.readLine() + System.lineSeparator(); + List lines = new ArrayList<>(); + String line = null; + while ((line = reader.readLine()) != null) { + lines.add(line); + } + + if (lines.size() <= subBatchSize) { + exchange.setProperty(SUB_BATCH_CREATED, false); + logger.info("Skipping creating sub batch, as batch size is less than configured sub-batch size"); + return; + } + + List subBatchFile = new ArrayList<>(); + int subBatchCount = 1; + for (int i = 0; i < lines.size(); i+=subBatchSize) { + String filename = "sub-batch-"+subBatchCount+".csv"; + FileWriter writer = new FileWriter(filename); + writer.write(header); + for (int j = i; j < Math.min(i + subBatchSize, lines.size()); j++) { + writer.write(lines.get(j) + System.lineSeparator()); + } + writer.close(); + logger.info("Created sub-batch with file name {}", filename); + subBatchFile.add(filename); + subBatchCount++; + } + exchange.setProperty(SUB_BATCH_FILE_ARRAY, subBatchFile); + exchange.setProperty(SUB_BATCH_COUNT, subBatchFile.size()); + exchange.setProperty(SUB_BATCH_CREATED, true); + exchange.setProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, new ArrayList()); + }); + + /** + * Iterate through each CSVs of sub-batches and uploads in cloud + */ + from("direct:upload-sub-batch-file") + .id("direct:upload-sub-batch-file") + .log("Starting upload of sub-batch file") + .loopDoWhile(exchange -> exchange.getProperty(SUB_BATCH_FILE_ARRAY, List.class).size() > 0) + .process(exchange -> { + List subBatchFile = exchange.getProperty(SUB_BATCH_FILE_ARRAY, List.class); + exchange.setProperty(LOCAL_FILE_PATH, subBatchFile.remove(0)); + exchange.setProperty(SUB_BATCH_FILE_ARRAY, subBatchFile); + }) + .to("direct:upload-file") + .process(exchange -> { + String serverFilename = exchange.getProperty(SERVER_FILE_NAME, String.class); + List serverSubBatchFile = exchange.getProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, List.class); + serverSubBatchFile.add(serverFilename); + exchange.setProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, serverSubBatchFile); + }); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index b031db71..7cfd5632 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -31,4 +31,10 @@ private ZeebeVariables() { public static final String SPLITTING_FAILED = "splittingFailed"; public static final String FORMATTING_FAILED = "formattingFailed"; public static final String MERGE_FAILED = "mergeFailed"; + + public static final String FILE_NAME = "filename"; + + public static final String REQUEST_ID = "requestId"; + + public static final String SUB_BATCHES = "subBatches"; } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BaseWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BaseWorker.java index fe0d7abe..c225d24d 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BaseWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BaseWorker.java @@ -33,7 +33,7 @@ public abstract class BaseWorker { private int workerMaxJobs; @Autowired - private CamelContext camelContext; + protected CamelContext camelContext; @Autowired private ProducerTemplate producerTemplate; diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java index f16c02ae..06e2cf58 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java @@ -1,20 +1,56 @@ package org.mifos.processor.bulk.zeebe.worker; +import org.apache.camel.Exchange; +import org.apache.camel.support.DefaultExchange; +import org.mifos.processor.bulk.camel.routes.RouteId; +import org.springframework.stereotype.Component; + +import java.util.List; import java.util.Map; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_FAILED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_FAILED; +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; +@Component public class SplittingWorker extends BaseWorker { @Override public void setup() { + + /** + * This worker performs below tasks + * 1. Downloads the original CSV from cloud + * 2. Splits entire CSV into multiple CSV of sub-batches, based on configured sub-batch size. + * 3. Uploads the sub-batch CSVs to cloud + * 4. Sets zeebeVariable [SPLITTING_FAILED, SUB_BATCHES, SUB_BATCH_CREATED] + */ newWorker(Worker.SPLITTING, (client, job) -> { Map variables = job.getVariablesAsMap(); if (workerConfig.isSplittingWorkerEnabled) { variables.put(SPLITTING_FAILED, false); } + String filename = (String) variables.get(FILE_NAME); + Exchange exchange = new DefaultExchange(camelContext); + exchange.setProperty(SERVER_FILE_NAME, filename); + + try { + sendToCamelRoute(RouteId.SPLITTING, exchange); + assert !exchange.getProperty(SPLITTING_FAILED, Boolean.class); + } catch (Exception e) { + variables.put(SPLITTING_FAILED, true); + } + + Boolean subBatchCreated = exchange.getProperty(SUB_BATCH_CREATED, Boolean.class); + List serverSubBatchFile = exchange.getProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, List.class); + if (!subBatchCreated && serverSubBatchFile.isEmpty()) { + // if no sub-batches is created, insert the original filename in sub batch array + serverSubBatchFile.add(filename); + } + + variables.put(SUB_BATCHES, serverSubBatchFile); + variables.put(SUB_BATCH_CREATED, subBatchCreated); + client.newCompleteCommand(job.getKey()).variables(variables).send(); }); } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 3a9bd630..ab189bc9 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -67,7 +67,7 @@ config: enable: true splitting: enable: false - batch-size: 10 + sub-batch-size: 5 formatting: enable: true mergeback: From 5452bcd82b9c468bdae751a44ca30b7cb3e883c5 Mon Sep 17 00:00:00 2001 From: DanishJamal Date: Thu, 4 Aug 2022 16:25:02 +0530 Subject: [PATCH 020/156] Added worker enabling zeebe variable --- .../camel/routes/ProcessorStartRoute.java | 26 +++++++++++-------- .../processor/bulk/zeebe/ZeebeVariables.java | 15 +++++++++++ .../bulk/zeebe/worker/WorkerConfig.java | 2 ++ src/main/resources/application.yaml | 3 +++ 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 90fe0087..bcf9a8ae 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -1,30 +1,24 @@ package org.mifos.processor.bulk.camel.routes; -import io.undertow.util.MultipartParser; -import org.apache.camel.attachment.Attachment; -import org.apache.camel.attachment.AttachmentMessage; import org.mifos.processor.bulk.file.FileTransferService; import org.mifos.processor.bulk.zeebe.ZeebeProcessStarter; +import org.mifos.processor.bulk.zeebe.worker.WorkerConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.client.MultipartBodyBuilder; import org.springframework.stereotype.Component; -import org.springframework.web.multipart.MultipartFile; - import java.io.File; import java.io.FileWriter; -import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.UUID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; @Component -public class ProcessorStartRoute extends BaseRouteBuilder{ +public class ProcessorStartRoute extends BaseRouteBuilder { @Autowired private ZeebeProcessStarter zeebeProcessStarter; @@ -33,6 +27,9 @@ public class ProcessorStartRoute extends BaseRouteBuilder{ @Qualifier("awsStorage") private FileTransferService fileTransferService; + @Autowired + protected WorkerConfig workerConfig; + @Value("${application.bucket-name}") private String bucketName; @@ -77,8 +74,15 @@ private void setup() { Map variables = new HashMap<>(); variables.put(BATCH_ID, batchId); - variables.put("fileName", fileName); - variables.put("requestId", requestId); + variables.put(FILE_NAME, fileName); + variables.put(REQUEST_ID, requestId); + variables.put(PARTY_LOOKUP_ENABLED, workerConfig.isPartyLookUpWorkerEnabled); + variables.put(APPROVAL_ENABLED, workerConfig.isApprovalWorkerEnabled); + variables.put(ORDERING_ENABLED, workerConfig.isOrderingWorkerEnabled); + variables.put(SPLITTING_ENABLED, workerConfig.isSplittingWorkerEnabled); + variables.put(FORMATTING_ENABLED, workerConfig.isFormattingWorkerEnabled); + variables.put(SUCCESS_THRESHOLD_CHECK_ENABLED, workerConfig.isSuccessThresholdCheckEnabled); + variables.put(MERGE_ENABLED, workerConfig.isMergeBackWorkerEnabled); zeebeProcessStarter.startZeebeWorkflow(workflowId, "", variables); }); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 7cfd5632..0345c1c3 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -37,4 +37,19 @@ private ZeebeVariables() { public static final String REQUEST_ID = "requestId"; public static final String SUB_BATCHES = "subBatches"; + + public static final String PARTY_LOOKUP_ENABLED = "partyLookupEnabled"; + + public static final String APPROVAL_ENABLED = "approvalEnabled"; + + public static final String ORDERING_ENABLED = "orderingEnabled"; + + public static final String SPLITTING_ENABLED = "splittingEnabled"; + + public static final String FORMATTING_ENABLED = "formattingEnabled"; + + public static final String SUCCESS_THRESHOLD_CHECK_ENABLED = "successThresholdCheckEnabled"; + + public static final String MERGE_ENABLED = "mergeEnabled"; + } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java index 0eab8ecd..1fc0de7a 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java @@ -24,5 +24,7 @@ public class WorkerConfig { @Value("${config.mergeback.enable}") public boolean isMergeBackWorkerEnabled; + @Value("${config.success-threshold-check.enable}") + public boolean isSuccessThresholdCheckEnabled; } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index ab189bc9..201c1077 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -74,3 +74,6 @@ config: enable: false backpressure: enable: false + success-threshold-check: + enable: false + success-rate: 95 # in percentage From b4fc3cbe200b6ba71a3a6683c029a6ec87994cf9 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Fri, 5 Aug 2022 12:21:21 +0530 Subject: [PATCH 021/156] Implemented ordering logic + minor code fixes (#14) --- .../bulk/ConfigurationValidator.java | 60 ++++++++ .../bulk/camel/config/CamelProperties.java | 1 + .../bulk/camel/routes/OrderingRoute.java | 143 ++++++++++++++++++ .../camel/routes/ProcessorStartRoute.java | 6 +- .../bulk/camel/routes/SplittingRoute.java | 11 +- .../bulk/file/AwsFileTransferImpl.java | 2 +- .../processor/bulk/schema/Transaction.java | 4 + .../processor/bulk/zeebe/ZeebeVariables.java | 2 + .../bulk/zeebe/worker/OrderingWorker.java | 31 +++- .../bulk/zeebe/worker/SplittingWorker.java | 1 + src/main/resources/application.yaml | 1 + 11 files changed, 250 insertions(+), 12 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java diff --git a/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java b/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java new file mode 100644 index 00000000..4c7ad65b --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java @@ -0,0 +1,60 @@ +package org.mifos.processor.bulk; + +import org.mifos.processor.bulk.schema.Transaction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; + +@Component +public class ConfigurationValidator { + + public Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Value("${config.ordering.field}") + private String orderingField; + + @Value("${config.success-threshold-check.success-rate}") + private int successRate; + + @PostConstruct + private void validate() { + validateOrderingConfig(); + validateSuccessThresholdConfig(); + } + + // validates the ordering configuration + private void validateOrderingConfig() { + List possibleOrderingFields = new ArrayList<>(); + + for (Field field: Transaction.class.getDeclaredFields()) { + possibleOrderingFields.add(field.getName()); + } + + if (!possibleOrderingFields.contains(orderingField)) { + throw new ConfigurationValidationException("Invalid ordering field, possible values are " + + String.join(",", possibleOrderingFields)); + } + } + + // validates the success threshold related configuration + private void validateSuccessThresholdConfig() { + if (successRate <= 0 || successRate > 100) { + throw new ConfigurationValidationException("Invalid success threshold value configured (value=" + successRate + ")."); + } + + if (successRate < 50) { + logger.warn("It is advised to set the success threshold greater than 50. Currently configured as {}", successRate); + } + } + + // this exception is thrown when unexpected application config is set, and can't pass the ConfigurationValidator + public static class ConfigurationValidationException extends RuntimeException { + ConfigurationValidationException(String message) { super(message); } + } +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index f1f85017..96465c22 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -19,4 +19,5 @@ private CamelProperties() {} public static final String SERVER_SUB_BATCH_FILE_NAME_ARRAY = "serverSubBatchFileName"; + public static final String TRANSACTION_LIST = "transactionList"; } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java new file mode 100644 index 00000000..a9d296f8 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java @@ -0,0 +1,143 @@ +package org.mifos.processor.bulk.camel.routes; + +import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import org.mifos.processor.bulk.schema.Transaction; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import java.io.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERED_BY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERING_FAILED; + +@Component +public class OrderingRoute extends BaseRouteBuilder { + + @Autowired + private CsvMapper csvMapper; + + @Value("${config.ordering.field}") + private String orderingField; + + + @Override + public void configure() { + + /** + * Base route for kicking off ordering logic. Performs below tasks. + * 1. Downloads the csv form cloud. + * 2. Builds the [Transaction] array using CsvMapper. + * 3. Re-order the array generated in step1 based on [orderingField]. + * 4. Update file with the updated data. + * 5. Uploads the updated file in cloud. + */ + from(RouteId.ORDERING.getValue()) + .id(RouteId.ORDERING.getValue()) + .log("Starting route " + RouteId.ORDERING.name()) + .to("direct:download-file") + .to("direct:get-transaction-array") + .to("direct:order-data") + .to("direct:update-file") + .to("direct:upload-file") + .process(exchange -> { + exchange.setProperty(ORDERING_FAILED, false); + exchange.setProperty(ORDERED_BY, orderingField); + }); + + // Parse the [Transaction] array from the csv file + from("direct:get-transaction-array") + .id("direct:get-transaction-array") + .log("Starting route direct:get-transaction-array") + .process(exchange -> { + String filename = exchange.getProperty(LOCAL_FILE_PATH, String.class); + CsvSchema schema = CsvSchema.emptySchema().withHeader(); + FileReader reader = new FileReader(filename); + MappingIterator readValues = csvMapper.readerWithSchemaFor(Transaction.class).with(schema).readValues(reader); + List transactionList = new ArrayList<>(); + while (readValues.hasNext()) { + Transaction current = readValues.next(); + transactionList.add(current); + } + reader.close(); + exchange.setProperty(TRANSACTION_LIST, transactionList); + }); + + // re-order the array of [Transaction] based on [orderingField] + from("direct:order-data") + .id("direct:order-data") + .log("Starting route direct:order-data") + .process(exchange -> { + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + HashMap> stringListHashMap = new HashMap<>(); + transactionList.forEach((transaction) -> { + String key; + switch (orderingField) { + case "id": + key = "" + transaction.getId(); + break; + case "request_id": + key = transaction.getRequest_id(); + break; + case "account_number": + key = transaction.getAccount_number(); + break; + case "amount": + key = transaction.getAmount(); + break; + case "currency": + key = transaction.getCurrency(); + break; + case "note": + key = transaction.getNote(); + break; + default: + key = transaction.getPayment_mode(); + break; + } + + if (stringListHashMap.containsKey(key)) { + stringListHashMap.get(key).add(transaction); + } else { + stringListHashMap.put(key, new ArrayList() {{ + add(transaction); + }}); + } + }); + transactionList.clear(); + stringListHashMap.forEach((s, transactions) -> transactionList.addAll(transactions)); + exchange.setProperty(TRANSACTION_LIST, transactionList); + }); + + // updates the data in local file + from("direct:update-file") + .id("direct:update-file") + .log("Starting route direct:update-file") + .process(exchange -> { + String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + + // getting header + BufferedReader reader = new BufferedReader(new FileReader(filepath)); + String header = reader.readLine() + System.lineSeparator(); + reader.close(); + + FileWriter writer = new FileWriter(filepath); + writer.write(header); + + // updating file with re-ordered data + transactionList.forEach(transaction -> { + try { + writer.write(transaction.getCsvString() + System.lineSeparator()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + writer.close(); + }); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index bcf9a8ae..959e1ad0 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -36,10 +36,10 @@ public class ProcessorStartRoute extends BaseRouteBuilder { @Value("${bpmn.flows.bulk-processor}") private String workflowId; - private Logger logger = LoggerFactory.getLogger(this.getClass()); + private final Logger logger = LoggerFactory.getLogger(this.getClass()); @Override - public void configure() throws Exception { + public void configure() { setup(); } @@ -47,7 +47,7 @@ private void setup() { from("rest:POST:/bulk/transfer/{requestId}/{fileName}") .unmarshal().mimeMultipart("multipart/*") .process(exchange -> { - String fileName = exchange.getIn().getHeader("fileName", String.class); + String fileName = System.currentTimeMillis() + "_" + exchange.getIn().getHeader("fileName", String.class); String requestId = exchange.getIn().getHeader("requestId", String.class); String batchId = UUID.randomUUID().toString(); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java index 4dd6b691..ca3f0c9c 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java @@ -28,7 +28,7 @@ public void configure() throws Exception { from(RouteId.SPLITTING.getValue()) .id(RouteId.SPLITTING.getValue()) .log("Starting route " + RouteId.SPLITTING.name()) - .process(exchange -> exchange.setProperty(LOCAL_FILE_PATH, "1659424507931_ph-ee-bulk-demo-5.csv")) + .to("direct:download-file") .to("direct:create-sub-batch-file") .choice() .when(exchange -> exchange.getProperty(SUB_BATCH_CREATED, Boolean.class)) @@ -38,9 +38,7 @@ public void configure() throws Exception { .end() .process(exchange -> exchange.setProperty(SPLITTING_FAILED, false)); - /** - * Creates the sub-batch CSVs - */ + //Creates the sub-batch CSVs from("direct:create-sub-batch-file") .id("direct:create-sub-batch-file") .log("Creating sub-batch file") @@ -53,6 +51,7 @@ public void configure() throws Exception { while ((line = reader.readLine()) != null) { lines.add(line); } + reader.close(); if (lines.size() <= subBatchSize) { exchange.setProperty(SUB_BATCH_CREATED, false); @@ -80,9 +79,7 @@ public void configure() throws Exception { exchange.setProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, new ArrayList()); }); - /** - * Iterate through each CSVs of sub-batches and uploads in cloud - */ + // Iterate through each CSVs of sub-batches and uploads in cloud from("direct:upload-sub-batch-file") .id("direct:upload-sub-batch-file") .log("Starting upload of sub-batch file") diff --git a/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java b/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java index 05b14b27..7b68358c 100644 --- a/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java +++ b/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java @@ -33,7 +33,7 @@ public String uploadFile(MultipartFile file, String bucketName) { @Override public String uploadFile(File file, String bucketName) { - String fileName = System.currentTimeMillis() + "_" + file.getName(); + String fileName = file.getName(); s3Client.putObject(new PutObjectRequest(bucketName, fileName, file)); file.delete(); diff --git a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java index bdaf49d3..a9931d5a 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java +++ b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java @@ -92,4 +92,8 @@ public String toString() { ", batchId='" + batchId + '\'' + '}'; } + + public String getCsvString() { + return String.format("%s,%s,%s,%s,%s,%s,%s", id, request_id, payment_mode, account_number, amount, currency, note); + } } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 0345c1c3..f65b362e 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -52,4 +52,6 @@ private ZeebeVariables() { public static final String MERGE_ENABLED = "mergeEnabled"; + public static final String ORDERED_BY = "orderedBy"; + } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java index 899ef522..b75c0f54 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java @@ -1,19 +1,48 @@ package org.mifos.processor.bulk.zeebe.worker; +import org.apache.camel.Exchange; +import org.apache.camel.support.DefaultExchange; +import org.mifos.processor.bulk.camel.routes.RouteId; +import org.springframework.stereotype.Component; + import java.util.Map; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERING_FAILED; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; +@Component public class OrderingWorker extends BaseWorker { @Override public void setup() { + + /** + * This worker is responsible for ordering the data set based on field configuration. Performs below tasks. + * 1. Downloads the file from cloud. + * 2. Parse the data into POJO. + * 3. Re-order the data based on field configured in application.yaml + * 4. Uploads the updated file in cloud + */ newWorker(Worker.ORDERING, (client, job) -> { Map variables = job.getVariablesAsMap(); if (workerConfig.isOrderingWorkerEnabled) { variables.put(ORDERING_FAILED, false); } + String filename = (String) variables.get(FILE_NAME); + Exchange exchange = new DefaultExchange(camelContext); + exchange.setProperty(SERVER_FILE_NAME, filename); + + try { + sendToCamelRoute(RouteId.ORDERING, exchange); + assert !exchange.getProperty(ORDERING_FAILED, Boolean.class); + } catch (Exception e) { + variables.put(ORDERING_FAILED, true); + } + + variables.put(ORDERING_FAILED, false); + variables.put(ORDERED_BY, exchange.getProperty(ORDERED_BY)); + client.newCompleteCommand(job.getKey()).variables(variables).send(); }); } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java index 06e2cf58..d548c951 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java @@ -48,6 +48,7 @@ public void setup() { serverSubBatchFile.add(filename); } + variables.put(SPLITTING_FAILED, false); variables.put(SUB_BATCHES, serverSubBatchFile); variables.put(SUB_BATCH_CREATED, subBatchCreated); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 201c1077..f76d74c1 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -65,6 +65,7 @@ config: enable: false ordering: enable: true + field: "payment_mode" splitting: enable: false sub-batch-size: 5 From f3fd62d3ff91cb3a911b363f5d9a8232d63af00c Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Fri, 5 Aug 2022 16:48:29 +0530 Subject: [PATCH 022/156] Formatting worker implemented (#15) * Implemented ordering logic + minor code fixes * Fomratting worker implemented + minor refactoring --- .../bulk/ConfigurationValidator.java | 28 +++++- .../bulk/camel/config/CamelProperties.java | 2 + .../camel/routes/FileProcessingRoute.java | 88 +++++++++++++++++++ .../bulk/camel/routes/FileRoute.java | 1 + .../bulk/camel/routes/FormattingRoute.java | 76 ++++++++++++++++ .../bulk/camel/routes/OrderingRoute.java | 53 ----------- .../camel/routes/ProcessorStartRoute.java | 5 +- .../processor/bulk/schema/CsvSchema.java | 9 ++ .../bulk/schema/GSMATransaction.java | 74 ++++++++++++++++ .../processor/bulk/schema/Transaction.java | 9 +- .../processor/bulk/zeebe/ZeebeVariables.java | 1 + .../bulk/zeebe/worker/FormattingWorker.java | 32 ++++++- .../org/mifos/processor/format/Standard.java | 19 ++++ .../mifos/processor/format/StandardValue.java | 7 ++ .../processor/format/helper/BaseMapper.java | 19 ++++ .../processor/format/helper/GSMAMapper.java | 22 +++++ .../mifos/processor/format/helper/Mapper.java | 24 +++++ .../processor/format/helper/Mappers.java | 12 +++ src/main/resources/application.yaml | 1 + 19 files changed, 421 insertions(+), 61 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java create mode 100644 src/main/java/org/mifos/processor/bulk/schema/CsvSchema.java create mode 100644 src/main/java/org/mifos/processor/bulk/schema/GSMATransaction.java create mode 100644 src/main/java/org/mifos/processor/format/Standard.java create mode 100644 src/main/java/org/mifos/processor/format/StandardValue.java create mode 100644 src/main/java/org/mifos/processor/format/helper/BaseMapper.java create mode 100644 src/main/java/org/mifos/processor/format/helper/GSMAMapper.java create mode 100644 src/main/java/org/mifos/processor/format/helper/Mapper.java create mode 100644 src/main/java/org/mifos/processor/format/helper/Mappers.java diff --git a/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java b/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java index 4c7ad65b..6b9b62da 100644 --- a/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java +++ b/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java @@ -1,11 +1,11 @@ package org.mifos.processor.bulk; import org.mifos.processor.bulk.schema.Transaction; +import org.mifos.processor.format.Standard; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; - import javax.annotation.PostConstruct; import java.lang.reflect.Field; import java.util.ArrayList; @@ -22,10 +22,32 @@ public class ConfigurationValidator { @Value("${config.success-threshold-check.success-rate}") private int successRate; + @Value("${config.formatting.standard}") + private String standard; + @PostConstruct private void validate() { validateOrderingConfig(); validateSuccessThresholdConfig(); + validateFormattingStandard(); + } + + // validates the standard to be used for formatting + private void validateFormattingStandard() { + String std = this.standard.toUpperCase(); + try { + Standard standardEnum = Standard.valueOf(std); + logger.info("Configured formatting standard as >> {}", standardEnum.name()); + return; + } catch (Exception e) { + e.printStackTrace(); + } + List possibleStandards = new ArrayList<>(); + for (Field f: Standard.class.getFields()) { + possibleStandards.add(f.getName()); + } + throw new ConfigurationValidationException("Invalid standard configured for formatting data. Possible values are [" + + String.join(",", possibleStandards) + "]"); } // validates the ordering configuration @@ -37,8 +59,8 @@ private void validateOrderingConfig() { } if (!possibleOrderingFields.contains(orderingField)) { - throw new ConfigurationValidationException("Invalid ordering field, possible values are " - + String.join(",", possibleOrderingFields)); + throw new ConfigurationValidationException("Invalid ordering field, possible values are [" + + String.join(",", possibleOrderingFields) + "]"); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 96465c22..a93c7d7d 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -20,4 +20,6 @@ private CamelProperties() {} public static final String SERVER_SUB_BATCH_FILE_NAME_ARRAY = "serverSubBatchFileName"; public static final String TRANSACTION_LIST = "transactionList"; + + public static final String OVERRIDE_HEADER = "overrideHeader"; } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java new file mode 100644 index 00000000..53146c6a --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java @@ -0,0 +1,88 @@ +package org.mifos.processor.bulk.camel.routes; + +import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import org.mifos.processor.bulk.schema.Transaction; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; + +@Component +public class FileProcessingRoute extends BaseRouteBuilder { + + @Autowired + private CsvMapper csvMapper; + + @Override + public void configure() { + + /** + * Parse the [Transaction] array from the csv file + * exchangeInput: [LOCAL_FILE_PATH] the absolute path to the csv file + * exchangeOutput: [TRANSACTION_LIST] containing the list of [Transaction] + */ + from("direct:get-transaction-array") + .id("direct:get-transaction-array") + .log("Starting route direct:get-transaction-array") + .process(exchange -> { + String filename = exchange.getProperty(LOCAL_FILE_PATH, String.class); + CsvSchema schema = CsvSchema.emptySchema().withHeader(); + FileReader reader = new FileReader(filename); + MappingIterator readValues = csvMapper.readerWithSchemaFor(Transaction.class).with(schema).readValues(reader); + List transactionList = new ArrayList<>(); + while (readValues.hasNext()) { + Transaction current = readValues.next(); + transactionList.add(current); + } + reader.close(); + exchange.setProperty(TRANSACTION_LIST, transactionList); + }); + + /** + * updates the data in local file + * exchangeInput: + * [LOCAL_FILE_PATH] the absolute path to the csv file + * [TRANSACTION_LIST] containing the list of [Transaction] + * [OVERRIDE_HEADER] if set to true will override the header or else use the existing once in csv file + */ + from("direct:update-file") + .id("direct:update-file") + .log("Starting route direct:update-file") + .process(exchange -> { + String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + + // getting header + Boolean overrideHeader = exchange.getProperty(OVERRIDE_HEADER, Boolean.class); + String header; + if (overrideHeader == null || !overrideHeader) { + BufferedReader reader = new BufferedReader(new FileReader(filepath)); + header = reader.readLine() + System.lineSeparator(); + reader.close(); + } else { + header = transactionList.get(0).getCsvHeader() + System.lineSeparator(); + } + + FileWriter writer = new FileWriter(filepath); + writer.write(header); + + // updating file with re-ordered data + transactionList.forEach(transaction -> { + try { + writer.write(transaction.getCsvString() + System.lineSeparator()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + writer.close(); + logger.info("Update complete"); + }); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java index a204c06c..0f0a1542 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java @@ -42,6 +42,7 @@ public void configure() throws Exception { fos.write(csvFile); } exchange.setProperty(LOCAL_FILE_PATH, file.getAbsolutePath()); + logger.info("File downloaded"); }); /** diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java new file mode 100644 index 00000000..01373b19 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java @@ -0,0 +1,76 @@ +package org.mifos.processor.bulk.camel.routes; + +import org.mifos.processor.bulk.schema.Transaction; +import org.mifos.processor.format.Standard; +import org.mifos.processor.format.helper.Mappers; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import java.util.List; +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FORMATTING_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FORMATTING_STANDARD; + +@Component +public class FormattingRoute extends BaseRouteBuilder { + + + @Autowired + private Mappers mappers; + + @Value("${config.formatting.standard}") + private String standard; + + private Standard formattingStandard; + + @Override + public void configure() throws Exception { + // parsing enum from application.yaml string + formattingStandard = Standard.valueOf(standard); + + /** + * Base route for kicking off ordering logic. Performs below tasks. + * 1. Downloads the csv form cloud. + * 2. Builds the [Transaction] array using [direct:get-transaction-array] route. + * 3. Format the data based on the configuration provided in application.yaml. @see [Standard.java] + * 4. Update file with the updated data. + * 5. Uploads the updated file in cloud. + */ + from(RouteId.FORMATTING.getValue()) + .id(RouteId.FORMATTING.getValue()) + .log("Starting route " + RouteId.FORMATTING.name()) + .choice() + .when(exchange -> formattingStandard != Standard.DEFAULT) + .process(exchange -> exchange.setProperty(SERVER_FILE_NAME, "ph-ee-bulk-demo-5.csv")) + .to("direct:download-file") + .to("direct:get-transaction-array") + .to("direct:format-data") + // making sure to override header as well, since data format is now updated + .process(exchange -> exchange.setProperty(OVERRIDE_HEADER, true)) + .to("direct:update-file") + .to("direct:upload-file") + .otherwise() + .log("Skipping formatting since standard is set to DEFAULT") + .end() + .process(exchange -> { + exchange.setProperty(FORMATTING_FAILED, false); + exchange.setProperty(FORMATTING_STANDARD, standard); + }); + + // formatting data based on configuration. Uses [Mappers] for converting data. + from("direct:format-data") + .id("direct:format-data") + .log("Starting route direct:format-data") + .process(exchange -> { + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + // replace with switch when multiple standards are added + if (formattingStandard == Standard.GSMA) { + logger.info("Formatting based on {} standard", formattingStandard.name()); + exchange.setProperty(TRANSACTION_LIST, mappers.gsmaMapper.convertList(transactionList)); + } else { + exchange.setProperty(TRANSACTION_LIST, transactionList); + } + }); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java index a9d296f8..2767880d 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java @@ -1,13 +1,8 @@ package org.mifos.processor.bulk.camel.routes; -import com.fasterxml.jackson.databind.MappingIterator; -import com.fasterxml.jackson.dataformat.csv.CsvMapper; -import com.fasterxml.jackson.dataformat.csv.CsvSchema; import org.mifos.processor.bulk.schema.Transaction; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import java.io.*; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -18,9 +13,6 @@ @Component public class OrderingRoute extends BaseRouteBuilder { - @Autowired - private CsvMapper csvMapper; - @Value("${config.ordering.field}") private String orderingField; @@ -49,24 +41,6 @@ public void configure() { exchange.setProperty(ORDERED_BY, orderingField); }); - // Parse the [Transaction] array from the csv file - from("direct:get-transaction-array") - .id("direct:get-transaction-array") - .log("Starting route direct:get-transaction-array") - .process(exchange -> { - String filename = exchange.getProperty(LOCAL_FILE_PATH, String.class); - CsvSchema schema = CsvSchema.emptySchema().withHeader(); - FileReader reader = new FileReader(filename); - MappingIterator readValues = csvMapper.readerWithSchemaFor(Transaction.class).with(schema).readValues(reader); - List transactionList = new ArrayList<>(); - while (readValues.hasNext()) { - Transaction current = readValues.next(); - transactionList.add(current); - } - reader.close(); - exchange.setProperty(TRANSACTION_LIST, transactionList); - }); - // re-order the array of [Transaction] based on [orderingField] from("direct:order-data") .id("direct:order-data") @@ -112,32 +86,5 @@ public void configure() { stringListHashMap.forEach((s, transactions) -> transactionList.addAll(transactions)); exchange.setProperty(TRANSACTION_LIST, transactionList); }); - - // updates the data in local file - from("direct:update-file") - .id("direct:update-file") - .log("Starting route direct:update-file") - .process(exchange -> { - String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); - List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); - - // getting header - BufferedReader reader = new BufferedReader(new FileReader(filepath)); - String header = reader.readLine() + System.lineSeparator(); - reader.close(); - - FileWriter writer = new FileWriter(filepath); - writer.write(header); - - // updating file with re-ordered data - transactionList.forEach(transaction -> { - try { - writer.write(transaction.getCsvString() + System.lineSeparator()); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - writer.close(); - }); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 959e1ad0..d961997c 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -45,7 +45,8 @@ public void configure() { private void setup() { from("rest:POST:/bulk/transfer/{requestId}/{fileName}") - .unmarshal().mimeMultipart("multipart/*") + .to(RouteId.FORMATTING.getValue()) + /*.unmarshal().mimeMultipart("multipart/*") .process(exchange -> { String fileName = System.currentTimeMillis() + "_" + exchange.getIn().getHeader("fileName", String.class); String requestId = exchange.getIn().getHeader("requestId", String.class); @@ -85,6 +86,6 @@ private void setup() { variables.put(MERGE_ENABLED, workerConfig.isMergeBackWorkerEnabled); zeebeProcessStarter.startZeebeWorkflow(workflowId, "", variables); - }); + })*/; } } diff --git a/src/main/java/org/mifos/processor/bulk/schema/CsvSchema.java b/src/main/java/org/mifos/processor/bulk/schema/CsvSchema.java new file mode 100644 index 00000000..95531f8a --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/CsvSchema.java @@ -0,0 +1,9 @@ +package org.mifos.processor.bulk.schema; + +public interface CsvSchema { + + String getCsvString(); + + String getCsvHeader(); + +} diff --git a/src/main/java/org/mifos/processor/bulk/schema/GSMATransaction.java b/src/main/java/org/mifos/processor/bulk/schema/GSMATransaction.java new file mode 100644 index 00000000..2d70c7a6 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/GSMATransaction.java @@ -0,0 +1,74 @@ +package org.mifos.processor.bulk.schema; + +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonPropertyOrder({ "id", "requestId", "paymentMode", "accountNumber", "amount", "note" }) +public class GSMATransaction implements CsvSchema { + + private int id; + private String requestId; + private String paymentMode; + private String accountNumber; + private String amount; + + private String note; + + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getAmount() { + return amount; + } + + public void setAmount(String amount) { + this.amount = amount; + } + + public String getNote() { + return note; + } + + public void setNote(String note) { + this.note = note; + } + + public String getRequestId() { + return requestId; + } + + public void setRequestId(String requestId) { + this.requestId = requestId; + } + + public String getPaymentMode() { + return paymentMode; + } + + public void setPaymentMode(String paymentMode) { + this.paymentMode = paymentMode; + } + + public String getAccountNumber() { + return accountNumber; + } + + public void setAccountNumber(String accountNumber) { + this.accountNumber = accountNumber; + } + + @Override + public String getCsvString() { + return String.format("%s,%s,%s,%s,%s,%s", id, requestId, paymentMode, accountNumber, amount, note); + } + + @Override + public String getCsvHeader() { + return "id,requestId,paymentMode,accountNumber,amount,note"; + } +} diff --git a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java index a9931d5a..bb387222 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java +++ b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java @@ -3,7 +3,7 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; @JsonPropertyOrder({ "id", "request_id", "payment_mode", "account_number", "amount", "currency", "note" }) -public class Transaction { +public class Transaction implements CsvSchema { private int id; private String request_id; @@ -93,7 +93,14 @@ public String toString() { '}'; } + @Override public String getCsvString() { return String.format("%s,%s,%s,%s,%s,%s,%s", id, request_id, payment_mode, account_number, amount, currency, note); } + + @Override + public String getCsvHeader() { + return "id,request_id,payment_mode,account_number,amount,currency,note"; + } + } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index f65b362e..7af50329 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -54,4 +54,5 @@ private ZeebeVariables() { public static final String ORDERED_BY = "orderedBy"; + public static final String FORMATTING_STANDARD = "formattingStandard"; } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/FormattingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/FormattingWorker.java index 53905404..d2ded17b 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/FormattingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/FormattingWorker.java @@ -1,19 +1,47 @@ package org.mifos.processor.bulk.zeebe.worker; -import java.util.Map; +import org.apache.camel.Exchange; +import org.apache.camel.support.DefaultExchange; +import org.mifos.processor.bulk.camel.routes.RouteId; +import org.springframework.stereotype.Component; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FORMATTING_FAILED; +import java.util.Map; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; +@Component public class FormattingWorker extends BaseWorker { @Override public void setup() { + + /** + * Starts the new worker for formatting of the data. Performs below tasks + * 1. Downloads the file from cloud. + * 2. Parse the data into POJO. + * 3. Format the data based on field configured in application.yaml + * 4. Uploads the updated file in cloud + */ newWorker(Worker.FORMATTING, (client, job) -> { Map variables = job.getVariablesAsMap(); if (workerConfig.isFormattingWorkerEnabled) { variables.put(FORMATTING_FAILED, false); } + String filename = (String) variables.get(FILE_NAME); + Exchange exchange = new DefaultExchange(camelContext); + exchange.setProperty(SERVER_FILE_NAME, filename); + + try { + sendToCamelRoute(RouteId.FORMATTING, exchange); + assert !exchange.getProperty(FORMATTING_FAILED, Boolean.class); + } catch (Exception e) { + variables.put(FORMATTING_FAILED, true); + } + + variables.put(FORMATTING_FAILED, false); + variables.put(FORMATTING_STANDARD, exchange.getProperty(FORMATTING_STANDARD)); + client.newCompleteCommand(job.getKey()).variables(variables).send(); }); } diff --git a/src/main/java/org/mifos/processor/format/Standard.java b/src/main/java/org/mifos/processor/format/Standard.java new file mode 100644 index 00000000..ff9b36dd --- /dev/null +++ b/src/main/java/org/mifos/processor/format/Standard.java @@ -0,0 +1,19 @@ +package org.mifos.processor.format; + +public enum Standard { + + GSMA(StandardValue.GSMA), + DEFAULT(StandardValue.DEFAULT); + + private final String value; + + Standard(String s) { + value = s; + } + + public String getValue() { + return value; + } + +} + diff --git a/src/main/java/org/mifos/processor/format/StandardValue.java b/src/main/java/org/mifos/processor/format/StandardValue.java new file mode 100644 index 00000000..55950043 --- /dev/null +++ b/src/main/java/org/mifos/processor/format/StandardValue.java @@ -0,0 +1,7 @@ +package org.mifos.processor.format; + +public class StandardValue { + public final static String GSMA = "GSMA"; + + public final static String DEFAULT = "DEFAULT"; +} diff --git a/src/main/java/org/mifos/processor/format/helper/BaseMapper.java b/src/main/java/org/mifos/processor/format/helper/BaseMapper.java new file mode 100644 index 00000000..7b628826 --- /dev/null +++ b/src/main/java/org/mifos/processor/format/helper/BaseMapper.java @@ -0,0 +1,19 @@ +package org.mifos.processor.format.helper; + +import org.mifos.processor.bulk.schema.CsvSchema; +import org.mifos.processor.bulk.schema.GSMATransaction; + +import java.util.ArrayList; +import java.util.List; + +public abstract class BaseMapper implements Mapper { + + @Override + public List convertList(List objects) { + List list = new ArrayList<>(); + objects.forEach(transaction -> { + list.add(convert(transaction)); + }); + return list; + } +} diff --git a/src/main/java/org/mifos/processor/format/helper/GSMAMapper.java b/src/main/java/org/mifos/processor/format/helper/GSMAMapper.java new file mode 100644 index 00000000..27497c38 --- /dev/null +++ b/src/main/java/org/mifos/processor/format/helper/GSMAMapper.java @@ -0,0 +1,22 @@ +package org.mifos.processor.format.helper; + +import org.mifos.processor.bulk.schema.GSMATransaction; +import org.mifos.processor.bulk.schema.Transaction; +import org.springframework.stereotype.Component; +import java.util.ArrayList; +import java.util.List; + +@Component +public class GSMAMapper extends BaseMapper { + @Override + public GSMATransaction convert(Transaction object) { + GSMATransaction gsmaTransaction = new GSMATransaction(); + gsmaTransaction.setId(object.getId()); + gsmaTransaction.setRequestId("test"); + gsmaTransaction.setPaymentMode(object.getPayment_mode()); + gsmaTransaction.setAccountNumber(object.getAccount_number()); + gsmaTransaction.setAmount(object.getAmount()); + gsmaTransaction.setNote(object.getNote()); + return gsmaTransaction; + } +} diff --git a/src/main/java/org/mifos/processor/format/helper/Mapper.java b/src/main/java/org/mifos/processor/format/helper/Mapper.java new file mode 100644 index 00000000..3cb06269 --- /dev/null +++ b/src/main/java/org/mifos/processor/format/helper/Mapper.java @@ -0,0 +1,24 @@ +package org.mifos.processor.format.helper; + +import org.mifos.processor.bulk.schema.CsvSchema; +import org.springframework.stereotype.Service; + +import java.util.List; + +public interface Mapper { + + /** + * Use for converting the object of type [FROM] to an object of type [TO] + * @param object of type [FROM] + * @return object of type [To] + */ + public TO convert(FROM object); + + /** + * Use for converting the list of objects of type [FROM] to an object of type [TO] + * @param objects lost of object of type [FROM] + * @return objects of type [To] + */ + public List convertList(List objects); + +} diff --git a/src/main/java/org/mifos/processor/format/helper/Mappers.java b/src/main/java/org/mifos/processor/format/helper/Mappers.java new file mode 100644 index 00000000..4cfdeb54 --- /dev/null +++ b/src/main/java/org/mifos/processor/format/helper/Mappers.java @@ -0,0 +1,12 @@ +package org.mifos.processor.format.helper; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class Mappers { + + @Autowired + public GSMAMapper gsmaMapper; + +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index f76d74c1..6109ce45 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -71,6 +71,7 @@ config: sub-batch-size: 5 formatting: enable: true + standard: "DEFAULT" mergeback: enable: false backpressure: From 05d366fff9bd24dc79055f7995df6eaa2e8d0f1d Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Tue, 30 Aug 2022 00:21:45 +0530 Subject: [PATCH 023/156] Implemented init-sub-batch worker (#16) * Implemented init-sub-batch worker * Removed csvs * Zeebe contactpoint updated --- .../bulk/ConfigurationValidator.java | 2 +- .../java/org/mifos/processor/bulk/Utils.java | 9 +++ .../bulk/camel/config/CamelProperties.java | 5 +- .../bulk/camel/routes/BaseRouteBuilder.java | 6 ++ .../bulk/camel/routes/FormattingRoute.java | 4 +- .../bulk/camel/routes/InitSubBatchRoute.java | 67 +++++++++++++++++ .../camel/routes/ProcessorStartRoute.java | 32 +++++++- .../processor/bulk/camel/routes/RouteId.java | 3 +- .../bulk/camel/routes/SplittingRoute.java | 3 +- .../processor/{ => bulk}/format/Standard.java | 2 +- .../{ => bulk}/format/StandardValue.java | 2 +- .../{ => bulk}/format/helper/BaseMapper.java | 3 +- .../{ => bulk}/format/helper/GSMAMapper.java | 4 +- .../{ => bulk}/format/helper/Mapper.java | 2 +- .../{ => bulk}/format/helper/Mappers.java | 2 +- .../processor/bulk/schema/Transaction.java | 3 + .../processor/bulk/zeebe/BpmnConfig.java | 12 +++ .../bulk/zeebe/ZeebeProcessStarter.java | 22 +++++- .../processor/bulk/zeebe/ZeebeVariables.java | 11 +++ .../bulk/zeebe/worker/InitSubBatchWorker.java | 75 +++++++++++++++++++ .../bulk/zeebe/worker/SplittingWorker.java | 11 ++- .../processor/bulk/zeebe/worker/Worker.java | 4 +- src/main/resources/application.yaml | 12 +-- 23 files changed, 266 insertions(+), 30 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/Utils.java create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java rename src/main/java/org/mifos/processor/{ => bulk}/format/Standard.java (85%) rename src/main/java/org/mifos/processor/{ => bulk}/format/StandardValue.java (76%) rename src/main/java/org/mifos/processor/{ => bulk}/format/helper/BaseMapper.java (82%) rename src/main/java/org/mifos/processor/{ => bulk}/format/helper/GSMAMapper.java (88%) rename src/main/java/org/mifos/processor/{ => bulk}/format/helper/Mapper.java (93%) rename src/main/java/org/mifos/processor/{ => bulk}/format/helper/Mappers.java (80%) create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/BpmnConfig.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java diff --git a/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java b/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java index 6b9b62da..8e36588a 100644 --- a/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java +++ b/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java @@ -1,7 +1,7 @@ package org.mifos.processor.bulk; import org.mifos.processor.bulk.schema.Transaction; -import org.mifos.processor.format.Standard; +import org.mifos.processor.bulk.format.Standard; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; diff --git a/src/main/java/org/mifos/processor/bulk/Utils.java b/src/main/java/org/mifos/processor/bulk/Utils.java new file mode 100644 index 00000000..02389136 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/Utils.java @@ -0,0 +1,9 @@ +package org.mifos.processor.bulk; + +public class Utils { + + public static String getTenantSpecificWorkflowId(String originalWorkflowName, String tenantName) { + return originalWorkflowName.replace("{dfspid}", tenantName); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index a93c7d7d..20083d79 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -7,7 +7,7 @@ private CamelProperties() {} public static final String AUTH_TYPE = "authType"; public static final String IS_BATCH_READY = "isBatchReady"; // camel property to check if batch is ready for sampling - public static final String SERVER_FILE_NAME = "serverFileNam"; + public static final String SERVER_FILE_NAME = "serverFileName"; public static final String LOCAL_FILE_PATH = "localFilePath"; @@ -22,4 +22,7 @@ private CamelProperties() {} public static final String TRANSACTION_LIST = "transactionList"; public static final String OVERRIDE_HEADER = "overrideHeader"; + + public static final String TENANT_NAME = "tenantName"; + } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java index 30574a00..0598a326 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java @@ -9,9 +9,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; + import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.List; public abstract class BaseRouteBuilder extends RouteBuilder { @@ -24,6 +27,9 @@ public abstract class BaseRouteBuilder extends RouteBuilder { @Autowired ZeebeClient zeebeClient; + @Value("#{'${tenants}'.split(',')}") + protected List tenants; + public Logger logger = LoggerFactory.getLogger(this.getClass()); public RouteDefinition getBaseExternalApiRequestRouteDefinition(String routeId, HttpRequestMethod httpMethod) { diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java index 01373b19..bd9f36e4 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java @@ -1,8 +1,8 @@ package org.mifos.processor.bulk.camel.routes; import org.mifos.processor.bulk.schema.Transaction; -import org.mifos.processor.format.Standard; -import org.mifos.processor.format.helper.Mappers; +import org.mifos.processor.bulk.format.Standard; +import org.mifos.processor.bulk.format.helper.Mappers; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java new file mode 100644 index 00000000..f3d1e394 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -0,0 +1,67 @@ +package org.mifos.processor.bulk.camel.routes; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.mifos.processor.bulk.Utils; +import org.mifos.processor.bulk.schema.Transaction; +import org.mifos.processor.bulk.zeebe.BpmnConfig; +import org.mifos.processor.bulk.zeebe.ZeebeProcessStarter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; + +@Component +public class InitSubBatchRoute extends BaseRouteBuilder { + + @Autowired + private ZeebeProcessStarter zeebeProcessStarter; + + @Autowired + private BpmnConfig bpmnConfig; + + @Override + public void configure() throws Exception { + + /** + * Base route for kicking off init sub batch logic. Performs below tasks. + * 1. Downloads the csv form cloud. + * 2. Builds the [Transaction] array using [direct:get-transaction-array] route. + * 3. Loops through each transaction and start the respective workflow + */ + from(RouteId.INIT_SUB_BATCH.getValue()) + .id(RouteId.INIT_SUB_BATCH.getValue()) + .log("Starting route " + RouteId.INIT_SUB_BATCH.name()) + .to("direct:download-file") + .to("direct:get-transaction-array") + .to("direct:start-workflow"); + + // Loops through each transaction and start the respective workflow + from("direct:start-workflow") + .id("direct:start-flow") + .log("Starting route direct:start-flow") + .process(exchange -> { + String tenantName = exchange.getProperty(TENANT_NAME, String.class); + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + if (transactionList.get(0).getPayment_mode().equalsIgnoreCase("slcb")) { + Map variables = new HashMap<>(); + variables.put(BATCH_ID, exchange.getProperty(BATCH_ID)); + variables.put(SUB_BATCH_ID, UUID.randomUUID().toString()); + variables.put(FILE_NAME, exchange.getProperty(SERVER_FILE_NAME)); + variables.put(REQUEST_ID, exchange.getProperty(REQUEST_ID)); + variables.put(PURPOSE, exchange.getProperty(PURPOSE)); + + zeebeProcessStarter.startZeebeWorkflow( + Utils.getTenantSpecificWorkflowId(bpmnConfig.slcbBpmn, tenantName), variables); + } + + exchange.setProperty(INIT_SUB_BATCH_FAILED, false); + }); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index d961997c..17cbd421 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -1,5 +1,6 @@ package org.mifos.processor.bulk.camel.routes; +import org.apache.camel.attachment.AttachmentMessage; import org.mifos.processor.bulk.file.FileTransferService; import org.mifos.processor.bulk.zeebe.ZeebeProcessStarter; import org.mifos.processor.bulk.zeebe.worker.WorkerConfig; @@ -11,9 +12,13 @@ import org.springframework.stereotype.Component; import java.io.File; import java.io.FileWriter; +import java.io.IOException; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.UUID; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @@ -45,13 +50,18 @@ public void configure() { private void setup() { from("rest:POST:/bulk/transfer/{requestId}/{fileName}") - .to(RouteId.FORMATTING.getValue()) - /*.unmarshal().mimeMultipart("multipart/*") + .unmarshal().mimeMultipart("multipart/*") + .to("direct:validate-tenant") .process(exchange -> { String fileName = System.currentTimeMillis() + "_" + exchange.getIn().getHeader("fileName", String.class); String requestId = exchange.getIn().getHeader("requestId", String.class); + String purpose = exchange.getIn().getHeader("purpose", String.class); String batchId = UUID.randomUUID().toString(); + if (purpose == null || purpose.isEmpty()) { + purpose = "test payment"; + } + logger.info("\n\n Filename: " + fileName + " \n\n"); logger.info("\n\n BatchId: " + batchId + " \n\n"); @@ -77,6 +87,8 @@ private void setup() { variables.put(BATCH_ID, batchId); variables.put(FILE_NAME, fileName); variables.put(REQUEST_ID, requestId); + variables.put(PURPOSE, purpose); + variables.put(TENANT_ID, exchange.getProperty(TENANT_NAME)); variables.put(PARTY_LOOKUP_ENABLED, workerConfig.isPartyLookUpWorkerEnabled); variables.put(APPROVAL_ENABLED, workerConfig.isApprovalWorkerEnabled); variables.put(ORDERING_ENABLED, workerConfig.isOrderingWorkerEnabled); @@ -86,6 +98,20 @@ private void setup() { variables.put(MERGE_ENABLED, workerConfig.isMergeBackWorkerEnabled); zeebeProcessStarter.startZeebeWorkflow(workflowId, "", variables); - })*/; + exchange.getIn().setBody(batchId); + }); + + from("direct:validate-tenant") + .id("direct:validate-tenant") + .log("Validating tenant") + .process(exchange -> { + String tenantName = exchange.getIn().getHeader("Platform-TenantId", String.class); + if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { + throw new Exception("Invalid tenant value."); + } + exchange.setProperty(TENANT_NAME, tenantName); + }); + + } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java b/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java index 051ca08f..8e182df0 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java @@ -8,7 +8,8 @@ public enum RouteId { SPLITTING("direct:splitting"), FORMATTING("direct:formatting"), BATCH_STATUS("direct:batchStatus"), - MERGE_BACK("direct:mergeSubBatch"); + MERGE_BACK("direct:mergeSubBatch"), + INIT_SUB_BATCH("direct:init-sub-batches"); private final String value; diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java index ca3f0c9c..d8e48043 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java @@ -7,7 +7,6 @@ import java.io.FileWriter; import java.util.ArrayList; import java.util.List; - import static org.mifos.processor.bulk.camel.config.CamelProperties.*; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_FAILED; @@ -62,7 +61,7 @@ public void configure() throws Exception { List subBatchFile = new ArrayList<>(); int subBatchCount = 1; for (int i = 0; i < lines.size(); i+=subBatchSize) { - String filename = "sub-batch-"+subBatchCount+".csv"; + String filename = System.currentTimeMillis() + "_" + "sub-batch-"+subBatchCount+".csv"; FileWriter writer = new FileWriter(filename); writer.write(header); for (int j = i; j < Math.min(i + subBatchSize, lines.size()); j++) { diff --git a/src/main/java/org/mifos/processor/format/Standard.java b/src/main/java/org/mifos/processor/bulk/format/Standard.java similarity index 85% rename from src/main/java/org/mifos/processor/format/Standard.java rename to src/main/java/org/mifos/processor/bulk/format/Standard.java index ff9b36dd..11d6e8de 100644 --- a/src/main/java/org/mifos/processor/format/Standard.java +++ b/src/main/java/org/mifos/processor/bulk/format/Standard.java @@ -1,4 +1,4 @@ -package org.mifos.processor.format; +package org.mifos.processor.bulk.format; public enum Standard { diff --git a/src/main/java/org/mifos/processor/format/StandardValue.java b/src/main/java/org/mifos/processor/bulk/format/StandardValue.java similarity index 76% rename from src/main/java/org/mifos/processor/format/StandardValue.java rename to src/main/java/org/mifos/processor/bulk/format/StandardValue.java index 55950043..a10e4ff6 100644 --- a/src/main/java/org/mifos/processor/format/StandardValue.java +++ b/src/main/java/org/mifos/processor/bulk/format/StandardValue.java @@ -1,4 +1,4 @@ -package org.mifos.processor.format; +package org.mifos.processor.bulk.format; public class StandardValue { public final static String GSMA = "GSMA"; diff --git a/src/main/java/org/mifos/processor/format/helper/BaseMapper.java b/src/main/java/org/mifos/processor/bulk/format/helper/BaseMapper.java similarity index 82% rename from src/main/java/org/mifos/processor/format/helper/BaseMapper.java rename to src/main/java/org/mifos/processor/bulk/format/helper/BaseMapper.java index 7b628826..0481fed5 100644 --- a/src/main/java/org/mifos/processor/format/helper/BaseMapper.java +++ b/src/main/java/org/mifos/processor/bulk/format/helper/BaseMapper.java @@ -1,7 +1,6 @@ -package org.mifos.processor.format.helper; +package org.mifos.processor.bulk.format.helper; import org.mifos.processor.bulk.schema.CsvSchema; -import org.mifos.processor.bulk.schema.GSMATransaction; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/org/mifos/processor/format/helper/GSMAMapper.java b/src/main/java/org/mifos/processor/bulk/format/helper/GSMAMapper.java similarity index 88% rename from src/main/java/org/mifos/processor/format/helper/GSMAMapper.java rename to src/main/java/org/mifos/processor/bulk/format/helper/GSMAMapper.java index 27497c38..4903a239 100644 --- a/src/main/java/org/mifos/processor/format/helper/GSMAMapper.java +++ b/src/main/java/org/mifos/processor/bulk/format/helper/GSMAMapper.java @@ -1,10 +1,8 @@ -package org.mifos.processor.format.helper; +package org.mifos.processor.bulk.format.helper; import org.mifos.processor.bulk.schema.GSMATransaction; import org.mifos.processor.bulk.schema.Transaction; import org.springframework.stereotype.Component; -import java.util.ArrayList; -import java.util.List; @Component public class GSMAMapper extends BaseMapper { diff --git a/src/main/java/org/mifos/processor/format/helper/Mapper.java b/src/main/java/org/mifos/processor/bulk/format/helper/Mapper.java similarity index 93% rename from src/main/java/org/mifos/processor/format/helper/Mapper.java rename to src/main/java/org/mifos/processor/bulk/format/helper/Mapper.java index 3cb06269..665785fc 100644 --- a/src/main/java/org/mifos/processor/format/helper/Mapper.java +++ b/src/main/java/org/mifos/processor/bulk/format/helper/Mapper.java @@ -1,4 +1,4 @@ -package org.mifos.processor.format.helper; +package org.mifos.processor.bulk.format.helper; import org.mifos.processor.bulk.schema.CsvSchema; import org.springframework.stereotype.Service; diff --git a/src/main/java/org/mifos/processor/format/helper/Mappers.java b/src/main/java/org/mifos/processor/bulk/format/helper/Mappers.java similarity index 80% rename from src/main/java/org/mifos/processor/format/helper/Mappers.java rename to src/main/java/org/mifos/processor/bulk/format/helper/Mappers.java index 4cfdeb54..351157ff 100644 --- a/src/main/java/org/mifos/processor/format/helper/Mappers.java +++ b/src/main/java/org/mifos/processor/bulk/format/helper/Mappers.java @@ -1,4 +1,4 @@ -package org.mifos.processor.format.helper; +package org.mifos.processor.bulk.format.helper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; diff --git a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java index bb387222..b2f76848 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java +++ b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java @@ -1,5 +1,6 @@ package org.mifos.processor.bulk.schema; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonPropertyOrder; @JsonPropertyOrder({ "id", "request_id", "payment_mode", "account_number", "amount", "currency", "note" }) @@ -93,11 +94,13 @@ public String toString() { '}'; } + @JsonIgnore @Override public String getCsvString() { return String.format("%s,%s,%s,%s,%s,%s,%s", id, request_id, payment_mode, account_number, amount, currency, note); } + @JsonIgnore @Override public String getCsvHeader() { return "id,request_id,payment_mode,account_number,amount,currency,note"; diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/BpmnConfig.java b/src/main/java/org/mifos/processor/bulk/zeebe/BpmnConfig.java new file mode 100644 index 00000000..f3238741 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/BpmnConfig.java @@ -0,0 +1,12 @@ +package org.mifos.processor.bulk.zeebe; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class BpmnConfig { + + @Value("${bpmn.flows.slcb}") + public String slcbBpmn; + +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeProcessStarter.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeProcessStarter.java index 5be27664..5f11ab4f 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeProcessStarter.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeProcessStarter.java @@ -42,6 +42,26 @@ public static void camelHeadersToZeebeVariables(Exchange exchange, Map extraVariables) { + + Map variables = new HashMap<>(); + variables.put(ZeebeVariables.ORIGIN_DATE, Instant.now().toEpochMilli()); + if(extraVariables != null) { + variables.putAll(extraVariables); + } + + // TODO if successful transfer response arrives in X timeout return it otherwise do callback + ProcessInstanceEvent join = zeebeClient.newCreateInstanceCommand() + .bpmnProcessId(workflowId) + .latestVersion() + .variables(variables) + .send() + .join(); + + + logger.info("zeebee workflow instance from process {}", workflowId); + } + public String startZeebeWorkflow(String workflowId, String request, Map extraVariables) { String transactionId = generateTransactionId(); @@ -70,4 +90,4 @@ public String startZeebeWorkflow(String workflowId, String request, Map { + Map variables = job.getVariablesAsMap(); + + List subBatches = (List) variables.get(SUB_BATCHES); + if (subBatches == null) { + subBatches = new ArrayList<>(); + } + List successSubBatches = (List) variables.get(INIT_SUCCESS_SUB_BATCHES); + if (successSubBatches == null) { + successSubBatches = new ArrayList<>(); + } + List failureSubBatches = (List) variables.get(INIT_FAILURE_SUB_BATCHES); + if (failureSubBatches == null) { + failureSubBatches = new ArrayList<>(); + } + boolean isSplittingEnabled = (boolean) variables.get(SPLITTING_ENABLED); + + if (!isSplittingEnabled) { + subBatches.add((String) variables.get(FILE_NAME)); + } + + String fileName = subBatches.remove(0); + + Exchange exchange = new DefaultExchange(camelContext); + exchange.setProperty(TENANT_NAME, variables.get(TENANT_ID)); + exchange.setProperty(SERVER_FILE_NAME, fileName); + exchange.setProperty(BATCH_ID, variables.get(BATCH_ID)); + exchange.setProperty(REQUEST_ID, variables.get(REQUEST_ID)); + exchange.setProperty(PURPOSE, variables.get(PURPOSE)); + + sendToCamelRoute(RouteId.INIT_SUB_BATCH, exchange); + + boolean subBatchFailed = exchange.getProperty(INIT_SUB_BATCH_FAILED, Boolean.class); + if (subBatchFailed) { + failureSubBatches.add(fileName); + } else { + successSubBatches.add(fileName); + } + + variables.put(REMAINING_SUB_BATCH, subBatches.size()); + variables.put(SUB_BATCHES, subBatches); + variables.put(INIT_SUCCESS_SUB_BATCHES, successSubBatches); + variables.put(INIT_FAILURE_SUB_BATCHES, failureSubBatches); + + client.newCompleteCommand(job.getKey()).variables(variables).send(); + }); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java index d548c951..06a4e328 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java @@ -5,6 +5,7 @@ import org.mifos.processor.bulk.camel.routes.RouteId; import org.springframework.stereotype.Component; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -42,14 +43,16 @@ public void setup() { } Boolean subBatchCreated = exchange.getProperty(SUB_BATCH_CREATED, Boolean.class); - List serverSubBatchFile = exchange.getProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, List.class); - if (!subBatchCreated && serverSubBatchFile.isEmpty()) { + List serverSubBatchFileList = exchange.getProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, List.class); + if (!subBatchCreated && serverSubBatchFileList.isEmpty()) { // if no sub-batches is created, insert the original filename in sub batch array - serverSubBatchFile.add(filename); + serverSubBatchFileList.add(filename); } variables.put(SPLITTING_FAILED, false); - variables.put(SUB_BATCHES, serverSubBatchFile); + variables.put(SUB_BATCHES, serverSubBatchFileList); + variables.put(INIT_SUCCESS_SUB_BATCHES, new ArrayList()); + variables.put(INIT_FAILURE_SUB_BATCHES, new ArrayList()); variables.put(SUB_BATCH_CREATED, subBatchCreated); client.newCompleteCommand(job.getKey()).variables(variables).send(); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java index df6fd2c3..c99a6221 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java @@ -8,7 +8,9 @@ public enum Worker { SPLITTING("splitting"), FORMATTING("formatting"), BATCH_STATUS("batchStatus"), - MERGE_BACK("mergeSubBatch"); + MERGE_BACK("mergeSubBatch"), + + INIT_SUB_BATCH("initSubBatch"); private final String value; diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 6109ce45..8d95673f 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -6,6 +6,7 @@ camel: json-jackson: auto-discover-object-mapper: true + application: bucket-name: paymenthub-ee-dev @@ -40,7 +41,7 @@ cloud: connection-string: -dfspids: "ibank-usa,ibank-india" +tenants: "ibank-usa,ibank-india" bpmn: flows: @@ -56,21 +57,22 @@ bpmn: international-remittance-payer: "international_remittance_payer_process-{dfspid}" debit-party-process: "debit_party_process-{dfspid}" bulk-processor: "bulk_processor-ibank-usa" + slcb: "slcb-{dfspid}" config: minimum-successful-tx-ratio: 0.90 partylookup: - enable: false + enable: true approval: - enable: false - ordering: enable: true + ordering: + enable: false field: "payment_mode" splitting: enable: false sub-batch-size: 5 formatting: - enable: true + enable: false standard: "DEFAULT" mergeback: enable: false From fb3afd9e163e206ac1c8f89f7ac0fafde123abe4 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Tue, 30 Aug 2022 21:54:22 +0530 Subject: [PATCH 024/156] Bug fixed in formattingRoute (#18) --- .../org/mifos/processor/bulk/camel/routes/FormattingRoute.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java index bd9f36e4..89bf9a1b 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java @@ -41,7 +41,6 @@ public void configure() throws Exception { .log("Starting route " + RouteId.FORMATTING.name()) .choice() .when(exchange -> formattingStandard != Standard.DEFAULT) - .process(exchange -> exchange.setProperty(SERVER_FILE_NAME, "ph-ee-bulk-demo-5.csv")) .to("direct:download-file") .to("direct:get-transaction-array") .to("direct:format-data") From 9e9524a24215a2204f00be4f84562865ad3d4982 Mon Sep 17 00:00:00 2001 From: DanishJamal Date: Thu, 1 Sep 2022 17:40:40 +0530 Subject: [PATCH 025/156] Added amount related variables as zeebe variable in bulk connectors --- .../processor/bulk/camel/routes/FileProcessingRoute.java | 9 +++++++++ .../processor/bulk/camel/routes/InitSubBatchRoute.java | 4 ++++ .../org/mifos/processor/bulk/zeebe/ZeebeVariables.java | 8 ++++++++ 3 files changed, 21 insertions(+) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java index 53146c6a..ebaafc2d 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java @@ -13,6 +13,7 @@ import java.util.ArrayList; import java.util.List; import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @Component public class FileProcessingRoute extends BaseRouteBuilder { @@ -32,6 +33,9 @@ public void configure() { .id("direct:get-transaction-array") .log("Starting route direct:get-transaction-array") .process(exchange -> { + Long totalAmount = 0L; + Long failedAmount = 0L; + Long completedAmount = 0L; String filename = exchange.getProperty(LOCAL_FILE_PATH, String.class); CsvSchema schema = CsvSchema.emptySchema().withHeader(); FileReader reader = new FileReader(filename); @@ -40,9 +44,14 @@ public void configure() { while (readValues.hasNext()) { Transaction current = readValues.next(); transactionList.add(current); + totalAmount += Long.parseLong(current.getAmount()); } reader.close(); exchange.setProperty(TRANSACTION_LIST, transactionList); + exchange.setProperty(TOTAL_AMOUNT, totalAmount); + exchange.setProperty(ONGOING_AMOUNT, totalAmount); // initially ongoing amount is same as total amount + exchange.setProperty(FAILED_AMOUNT, failedAmount); + exchange.setProperty(COMPLETED_AMOUNT, completedAmount); }); /** diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index f3d1e394..71acab09 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -55,6 +55,10 @@ public void configure() throws Exception { variables.put(FILE_NAME, exchange.getProperty(SERVER_FILE_NAME)); variables.put(REQUEST_ID, exchange.getProperty(REQUEST_ID)); variables.put(PURPOSE, exchange.getProperty(PURPOSE)); + variables.put(TOTAL_AMOUNT, exchange.getProperty(TOTAL_AMOUNT)); + variables.put(ONGOING_AMOUNT, exchange.getProperty(ONGOING_AMOUNT)); + variables.put(FAILED_AMOUNT, exchange.getProperty(FAILED_AMOUNT)); + variables.put(COMPLETED_AMOUNT, exchange.getProperty(COMPLETED_AMOUNT)); zeebeProcessStarter.startZeebeWorkflow( Utils.getTenantSpecificWorkflowId(bpmnConfig.slcbBpmn, tenantName), variables); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 3914124e..ea00b2b8 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -66,4 +66,12 @@ private ZeebeVariables() { public static final String REMAINING_SUB_BATCH = "remainingSubBatch"; public static final String TRANSACTION_REQUEST = "transactionRequest"; + + public static final String TOTAL_AMOUNT = "totalAmount"; + + public static final String ONGOING_AMOUNT = "ongoingAmount"; + + public static final String FAILED_AMOUNT = "failedAmount"; + + public static final String COMPLETED_AMOUNT = "completedAmount"; } From 77caed76c32032a1b1f345825565f7760bb8991c Mon Sep 17 00:00:00 2001 From: DanishJamal Date: Tue, 6 Sep 2022 18:57:10 +0530 Subject: [PATCH 026/156] Merge back logic implemented --- 1659424507931_ph-ee-bulk-demo-1.csv | 26 +++++ 1659424836231_ph-ee-bulk-demo-2.csv | 13 +++ .../java/org/mifos/processor/bulk/Utils.java | 9 -- .../bulk/camel/config/CamelProperties.java | 6 + .../bulk/camel/routes/InitSubBatchRoute.java | 3 +- .../bulk/camel/routes/MergeBackRoute.java | 109 ++++++++++++++++++ .../mifos/processor/bulk/utility/Utils.java | 35 ++++++ .../processor/bulk/zeebe/ZeebeVariables.java | 8 ++ .../bulk/zeebe/worker/MergeBackWorker.java | 42 ++++++- 9 files changed, 239 insertions(+), 12 deletions(-) create mode 100644 1659424507931_ph-ee-bulk-demo-1.csv create mode 100644 1659424836231_ph-ee-bulk-demo-2.csv delete mode 100644 src/main/java/org/mifos/processor/bulk/Utils.java create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java create mode 100644 src/main/java/org/mifos/processor/bulk/utility/Utils.java diff --git a/1659424507931_ph-ee-bulk-demo-1.csv b/1659424507931_ph-ee-bulk-demo-1.csv new file mode 100644 index 00000000..648314ea --- /dev/null +++ b/1659424507931_ph-ee-bulk-demo-1.csv @@ -0,0 +1,26 @@ +id,request_id,payment_mode,account_number,amount,currency,note +100,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment +101,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment +102,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,gsma,003001003874120160,850,USD,Test Payee Payment +103,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,mpesa,003001003873110196,222,USD,Test Payee Payment +104,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,gsma,003001003874120160,850,USD,Test Payee Payment +105,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,gsma,003001003873110196,222,USD,Test Payee Payment +106,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment +107,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,gsma,003001003873110196,222,USD,Test Payee Payment +108,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment +109,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,mpesa,003001003873110196,222,USD,Test Payee Payment +110,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,mpesa,003001003874120160,850,USD,Test Payee Payment +111,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment + +0,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment +1,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment +2,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment +3,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment +4,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment +5,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment +6,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment +7,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment +8,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment +9,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment +10,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment +11,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment diff --git a/1659424836231_ph-ee-bulk-demo-2.csv b/1659424836231_ph-ee-bulk-demo-2.csv new file mode 100644 index 00000000..fd3f40b2 --- /dev/null +++ b/1659424836231_ph-ee-bulk-demo-2.csv @@ -0,0 +1,13 @@ +id,request_id,payment_mode,account_number,amount,currency,note +0,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment +1,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment +2,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment +3,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment +4,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment +5,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment +6,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment +7,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment +8,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment +9,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment +10,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment +11,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment \ No newline at end of file diff --git a/src/main/java/org/mifos/processor/bulk/Utils.java b/src/main/java/org/mifos/processor/bulk/Utils.java deleted file mode 100644 index 02389136..00000000 --- a/src/main/java/org/mifos/processor/bulk/Utils.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.mifos.processor.bulk; - -public class Utils { - - public static String getTenantSpecificWorkflowId(String originalWorkflowName, String tenantName) { - return originalWorkflowName.replace("{dfspid}", tenantName); - } - -} diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 20083d79..98fb1891 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -11,6 +11,8 @@ private CamelProperties() {} public static final String LOCAL_FILE_PATH = "localFilePath"; + public static final String LOCAL_FILE_PATH_LIST = "localFilePaths"; + public static final String SUB_BATCH_FILE_ARRAY = "subBatchFileArray"; public static final String SUB_BATCH_COUNT = "subBatchCount"; @@ -25,4 +27,8 @@ private CamelProperties() {} public static final String TENANT_NAME = "tenantName"; + public static final String FILE_1 = "file1"; + + public static final String FILE_2 = "file2"; + } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index 71acab09..aa094b1f 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -1,7 +1,6 @@ package org.mifos.processor.bulk.camel.routes; -import com.fasterxml.jackson.core.JsonProcessingException; -import org.mifos.processor.bulk.Utils; +import org.mifos.processor.bulk.utility.Utils; import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.zeebe.BpmnConfig; import org.mifos.processor.bulk.zeebe.ZeebeProcessStarter; diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java new file mode 100644 index 00000000..b44f1dfc --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java @@ -0,0 +1,109 @@ +package org.mifos.processor.bulk.camel.routes; + +import org.mifos.processor.bulk.utility.Utils; +import org.springframework.stereotype.Component; +import java.io.File; +import java.util.List; +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; + +@Component +public class MergeBackRoute extends BaseRouteBuilder { + + + @Override + public void configure() throws Exception { + + /** + * Base route for kicking off merge back logic. Performs below tasks. + * 1. Picks the top two files from the array of files to be merged. + * 2. Merges them into single CSV. + * 3. Uploads the CSV to S3. + * 4. Updated the exchange variables. + */ + from(RouteId.MERGE_BACK.getValue()) + .id(RouteId.MERGE_BACK.getValue()) + .log("Starting route " + RouteId.MERGE_BACK.name()) + .choice() + .when(exchange -> exchange.getProperty(MERGE_FILE_LIST, List.class).size() == 0) + .log("Nothing to merge") + .setProperty(MERGE_FAILED, constant(false)) + .setProperty(MERGE_COMPLETED, constant(true)) + .when(exchange -> exchange.getProperty(MERGE_FILE_LIST, List.class).size() == 1) + .process(exchange -> { + exchange.setProperty(MERGE_FAILED, false); + exchange.setProperty(MERGE_COMPLETED, true); + exchange.setProperty(RESULT_FILE, exchange.getProperty(MERGE_FILE_LIST, List.class).get(0)); + }) + .otherwise() + .to("direct:start-merge") + .endChoice(); + + // starts the merge process, merges the file and uploads the file in s3 + from("direct:start-merge") + .id("direct:start-merge") + .log("Starting route direct:start-merge") + .to("direct:download-file-to-be-merged") + .process(exchange -> { + String file1 = exchange.getProperty(FILE_1, String.class); + String file2 = exchange.getProperty(FILE_2, String.class); + + String mergedFile = Utils.mergeCsvFile(file1, file2); + if (mergedFile == null) { + exchange.setProperty(MERGE_COMPLETED, false); + return; + } + if (exchange.getProperty(MERGE_ITERATION, Integer.class) == 1) { + // generate new name for merged file in case of first iteration + String newFileName = System.currentTimeMillis() + "_" + exchange.getProperty(BATCH_ID, String.class); + new File(mergedFile).renameTo(new File(newFileName)); + exchange.setProperty(LOCAL_FILE_PATH, newFileName); + } else { + exchange.setProperty(LOCAL_FILE_PATH, mergedFile); + } + }) + .to("direct:upload-file") + .process(exchange -> { + String mergedFileServerName = exchange.getProperty(SERVER_FILE_NAME, String.class); + List mergeList = exchange.getProperty(MERGE_FILE_LIST, List.class); + String first = mergeList.remove(0); + String second = mergeList.remove(0); + logger.info("Merge iteration {}, for list, {}", exchange.getProperty(MERGE_ITERATION), mergeList); + log.info("Merged files {} and {}", first, second); + mergeList.add(0, mergedFileServerName); + + + if (mergeList.size() == 1) { + exchange.setProperty(MERGE_FAILED, false); + exchange.setProperty(MERGE_COMPLETED, true); + exchange.setProperty(RESULT_FILE, mergedFileServerName); + } else { + exchange.setProperty(MERGE_COMPLETED, false); + } + + exchange.setProperty(MERGE_FILE_LIST, mergeList); + + // make sures to remove the files from local storage + new File(exchange.getProperty(FILE_1, String.class)).delete(); + new File(exchange.getProperty(FILE_2, String.class)).delete(); + }); + + // downloads the two files(using FIFO access pattern) from s3 which is to be merged. + from("direct:download-file-to-be-merged") + .id("direct:download-file-to-be-merged") + .log("Starting route direct:download-file-to-be-merged") + .log("Downloading files to be merged") + .process(exchange -> { + List mergeList = exchange.getProperty(MERGE_FILE_LIST, List.class); + exchange.setProperty(SERVER_FILE_NAME, mergeList.get(0)); + }) + .to("direct:download-file") // downloading first file + .setProperty(FILE_1, exchangeProperty(LOCAL_FILE_PATH)) + .process(exchange -> { + List mergeList = exchange.getProperty(MERGE_FILE_LIST, List.class); + exchange.setProperty(SERVER_FILE_NAME, mergeList.get(1)); + }) + .to("direct:download-file") // downloading second file + .setProperty(FILE_2, exchangeProperty(LOCAL_FILE_PATH)); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/utility/Utils.java b/src/main/java/org/mifos/processor/bulk/utility/Utils.java new file mode 100644 index 00000000..253258c5 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/utility/Utils.java @@ -0,0 +1,35 @@ +package org.mifos.processor.bulk.utility; + +import java.io.*; + +public class Utils { + + public static String getTenantSpecificWorkflowId(String originalWorkflowName, String tenantName) { + return originalWorkflowName.replace("{dfspid}", tenantName); + } + + public static String mergeCsvFile(String file1, String file2) { + try { + // create a writer for permFile + BufferedWriter out = new BufferedWriter(new FileWriter(file1, true)); + // create a reader for tmpFile + BufferedReader in = new BufferedReader(new FileReader(file2)); + String str; + int i = 0; + while ((str = in.readLine()) != null) { + if (i == 0) { + i = 1; + continue; + } + out.write(str+"\n"); + } + in.close(); + out.close(); + } catch (IOException e) { + return null; + } + + return file1; + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index ea00b2b8..6425d4b8 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -74,4 +74,12 @@ private ZeebeVariables() { public static final String FAILED_AMOUNT = "failedAmount"; public static final String COMPLETED_AMOUNT = "completedAmount"; + + public static final String MERGE_FILE_LIST = "mergeFiles"; + + public static final String MERGE_ITERATION = "mergeIteration"; + + public static final String MERGE_COMPLETED = "mergeCompleted"; + + public static final String RESULT_FILE = "resultFile"; } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java index 8f4a0fe8..a5d68a49 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java @@ -1,8 +1,14 @@ package org.mifos.processor.bulk.zeebe.worker; +import org.apache.camel.Exchange; +import org.apache.camel.support.DefaultExchange; +import org.mifos.processor.bulk.camel.routes.RouteId; + +import java.util.ArrayList; +import java.util.List; import java.util.Map; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; public class MergeBackWorker extends BaseWorker { @@ -14,6 +20,40 @@ public void setup() { variables.put(MERGE_FAILED, false); } + int mergeIteration = (int) variables.getOrDefault(MERGE_ITERATION, 1); + List subBatches = (List) variables.get(SUB_BATCHES); + List successSubBatches = (List) variables.get(INIT_SUCCESS_SUB_BATCHES); + List failureSubBatches = (List) variables.get(INIT_FAILURE_SUB_BATCHES); + + List mergeFileList = (List) variables.get(MERGE_FILE_LIST); + if (mergeFileList == null) { + mergeFileList = new ArrayList<>(); + mergeFileList.addAll(successSubBatches); + mergeFileList.addAll(failureSubBatches); + mergeFileList.addAll(subBatches); + } + + Exchange exchange = new DefaultExchange(camelContext); + exchange.setProperty(MERGE_FILE_LIST, mergeFileList); + exchange.setProperty(MERGE_ITERATION, mergeIteration); + exchange.setProperty(BATCH_ID, variables.get(BATCH_ID)); + + sendToCamelRoute(RouteId.MERGE_BACK, exchange); + + boolean mergeCompleted = exchange.getProperty(MERGE_COMPLETED, Boolean.class); + if (mergeCompleted) { + variables.put(MERGE_FAILED, exchange.getProperty(MERGE_FAILED, Boolean.class)); + String resultFile = exchange.getProperty(RESULT_FILE, String.class); + if (resultFile != null && !resultFile.isEmpty()) { + variables.put(RESULT_FILE, resultFile); + } + } + + variables.put(MERGE_FILE_LIST, exchange.getProperty(MERGE_FILE_LIST, List.class)); + variables.put(MERGE_COMPLETED, mergeCompleted); + variables.put(MERGE_ITERATION, ++mergeIteration); + + client.newCompleteCommand(job.getKey()).variables(variables).send(); }); } From 965e7b6e9931973d6e52fbb1287512053d8b6db7 Mon Sep 17 00:00:00 2001 From: DanishJamal Date: Tue, 6 Sep 2022 19:46:50 +0530 Subject: [PATCH 027/156] Minor fix --- .../mifos/processor/bulk/camel/routes/BaseRouteBuilder.java | 3 +++ .../mifos/processor/bulk/camel/routes/MergeBackRoute.java | 5 +++-- src/main/java/org/mifos/processor/bulk/utility/Utils.java | 4 ++++ .../org/mifos/processor/bulk/zeebe/worker/BaseWorker.java | 4 ++++ .../mifos/processor/bulk/zeebe/worker/MergeBackWorker.java | 4 ++++ src/main/resources/application.yaml | 1 + 6 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java index 0598a326..0e386213 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java @@ -30,6 +30,9 @@ public abstract class BaseRouteBuilder extends RouteBuilder { @Value("#{'${tenants}'.split(',')}") protected List tenants; + @Value("${cloud.aws.s3-base-url}") + protected String awsS3BaseUrl; + public Logger logger = LoggerFactory.getLogger(this.getClass()); public RouteDefinition getBaseExternalApiRequestRouteDefinition(String routeId, HttpRequestMethod httpMethod) { diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java index b44f1dfc..83a16598 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java @@ -55,7 +55,8 @@ public void configure() throws Exception { } if (exchange.getProperty(MERGE_ITERATION, Integer.class) == 1) { // generate new name for merged file in case of first iteration - String newFileName = System.currentTimeMillis() + "_" + exchange.getProperty(BATCH_ID, String.class); + String newFileName = System.currentTimeMillis() + "_" + + exchange.getProperty(BATCH_ID, String.class) + ".csv"; new File(mergedFile).renameTo(new File(newFileName)); exchange.setProperty(LOCAL_FILE_PATH, newFileName); } else { @@ -76,7 +77,7 @@ public void configure() throws Exception { if (mergeList.size() == 1) { exchange.setProperty(MERGE_FAILED, false); exchange.setProperty(MERGE_COMPLETED, true); - exchange.setProperty(RESULT_FILE, mergedFileServerName); + exchange.setProperty(RESULT_FILE, Utils.getAwsFileUrl(awsS3BaseUrl,mergedFileServerName)); } else { exchange.setProperty(MERGE_COMPLETED, false); } diff --git a/src/main/java/org/mifos/processor/bulk/utility/Utils.java b/src/main/java/org/mifos/processor/bulk/utility/Utils.java index 253258c5..5438ea95 100644 --- a/src/main/java/org/mifos/processor/bulk/utility/Utils.java +++ b/src/main/java/org/mifos/processor/bulk/utility/Utils.java @@ -32,4 +32,8 @@ public static String mergeCsvFile(String file1, String file2) { return file1; } + public static String getAwsFileUrl(String baseUrl, String filename) { + return String.format("%s/%s", baseUrl, filename); + } + } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BaseWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BaseWorker.java index c225d24d..8f35ebd1 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BaseWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BaseWorker.java @@ -8,6 +8,8 @@ import org.apache.camel.Exchange; import org.apache.camel.ProducerTemplate; import org.mifos.processor.bulk.camel.routes.RouteId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -41,6 +43,8 @@ public abstract class BaseWorker { @Autowired protected WorkerConfig workerConfig; + protected Logger logger = LoggerFactory.getLogger(this.getClass()); + @PostConstruct public abstract void setup(); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java index a5d68a49..3e41c86f 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java @@ -3,6 +3,7 @@ import org.apache.camel.Exchange; import org.apache.camel.support.DefaultExchange; import org.mifos.processor.bulk.camel.routes.RouteId; +import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.List; @@ -10,6 +11,7 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; +@Component public class MergeBackWorker extends BaseWorker { @Override @@ -38,6 +40,8 @@ public void setup() { exchange.setProperty(MERGE_ITERATION, mergeIteration); exchange.setProperty(BATCH_ID, variables.get(BATCH_ID)); + logger.info("Merge list: {}", mergeFileList); + sendToCamelRoute(RouteId.MERGE_BACK, exchange); boolean mergeCompleted = exchange.getProperty(MERGE_COMPLETED, Boolean.class); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 8d95673f..dfa92f32 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -28,6 +28,7 @@ operations-app: cloud: aws: enabled: true + s3-base-url: "https://paymenthub-ee-dev.s3.us-east-2.amazonaws.com" credentials: access-key: ${AWS_ACCESS_KEY:AKIAX32JM37TZOJ5AKFB} secret-key: ${AWS_SECRET_KEY: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} From d590beb2efdd604ed4cde05092222ec63ac797e5 Mon Sep 17 00:00:00 2001 From: DanishJamal Date: Wed, 7 Sep 2022 17:08:21 +0530 Subject: [PATCH 028/156] Batch status worker implemented --- 1659424507931_ph-ee-bulk-demo-1.csv | 26 --- 1659424836231_ph-ee-bulk-demo-2.csv | 13 -- build.gradle | 4 +- .../bulk/ConfigurationValidator.java | 30 +++- .../processor/bulk/OperationsAppConfig.java | 18 ++ .../bulk/camel/config/CamelContextConfig.java | 13 ++ .../bulk/camel/config/CamelProperties.java | 4 + .../HttpClientConfigurerTrustAllCACerts.java | 73 ++++++++ .../bulk/camel/routes/BaseRouteBuilder.java | 4 +- .../bulk/camel/routes/BatchStatusRoute.java | 87 ++++++++++ .../camel/routes/OperationsAuthRoute.java | 71 ++++++++ .../camel/routes/ProcessorStartRoute.java | 16 +- .../mifos/processor/bulk/schema/BatchDTO.java | 156 ++++++++++++++++++ .../mifos/processor/bulk/utility/Utils.java | 28 ++++ .../processor/bulk/zeebe/ZeebeVariables.java | 14 ++ .../bulk/zeebe/worker/BatchStatusWorker.java | 35 +++- src/main/resources/application.yaml | 11 +- 17 files changed, 551 insertions(+), 52 deletions(-) delete mode 100644 1659424507931_ph-ee-bulk-demo-1.csv delete mode 100644 1659424836231_ph-ee-bulk-demo-2.csv create mode 100644 src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/OperationsAuthRoute.java create mode 100644 src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java diff --git a/1659424507931_ph-ee-bulk-demo-1.csv b/1659424507931_ph-ee-bulk-demo-1.csv deleted file mode 100644 index 648314ea..00000000 --- a/1659424507931_ph-ee-bulk-demo-1.csv +++ /dev/null @@ -1,26 +0,0 @@ -id,request_id,payment_mode,account_number,amount,currency,note -100,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment -101,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment -102,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,gsma,003001003874120160,850,USD,Test Payee Payment -103,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,mpesa,003001003873110196,222,USD,Test Payee Payment -104,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,gsma,003001003874120160,850,USD,Test Payee Payment -105,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,gsma,003001003873110196,222,USD,Test Payee Payment -106,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment -107,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,gsma,003001003873110196,222,USD,Test Payee Payment -108,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment -109,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,mpesa,003001003873110196,222,USD,Test Payee Payment -110,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,mpesa,003001003874120160,850,USD,Test Payee Payment -111,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment - -0,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment -1,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment -2,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment -3,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment -4,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment -5,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment -6,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment -7,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment -8,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment -9,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment -10,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment -11,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment diff --git a/1659424836231_ph-ee-bulk-demo-2.csv b/1659424836231_ph-ee-bulk-demo-2.csv deleted file mode 100644 index fd3f40b2..00000000 --- a/1659424836231_ph-ee-bulk-demo-2.csv +++ /dev/null @@ -1,13 +0,0 @@ -id,request_id,payment_mode,account_number,amount,currency,note -0,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment -1,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment -2,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment -3,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment -4,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment -5,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment -6,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment -7,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment -8,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment -9,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment -10,f1e22fe3-9740-4fba-97b6-78f43bfa7f2f,slcb,003001003874120160,850,USD,Test Payee Payment -11,72aa3ea4-e6f6-4880-877f-39f6ac4d052e,slcb,003001003873110196,222,USD,Test Payee Payment \ No newline at end of file diff --git a/build.gradle b/build.gradle index ae240d5f..d0905e90 100644 --- a/build.gradle +++ b/build.gradle @@ -32,13 +32,15 @@ dependencies { implementation 'org.apache.camel:camel-undertow:3.4.0' implementation 'org.springframework.boot:spring-boot-starter:2.5.2' implementation 'org.springframework.boot:spring-boot-starter-web:2.5.2' + implementation 'org.springframework:spring-web:5.3.19' + implementation 'org.apache.camel:camel-http:3.4.0' implementation 'com.amazonaws:aws-java-sdk:1.11.486' implementation 'com.azure:azure-storage-blob:12.12.0' implementation 'io.camunda:zeebe-client-java:1.1.0' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.12.0' implementation 'org.springframework.kafka:spring-kafka:2.5.8.RELEASE' testImplementation 'org.springframework.boot:spring-boot-starter-test:2.5.2' - + implementation 'org.apache.camel:camel-jackson:3.4.0' implementation 'org.apache.camel.springboot:camel-mail-starter:3.4.0' } diff --git a/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java b/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java index 8e36588a..38487ced 100644 --- a/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java +++ b/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java @@ -2,8 +2,10 @@ import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.format.Standard; +import org.mifos.processor.bulk.zeebe.worker.WorkerConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; @@ -19,17 +21,37 @@ public class ConfigurationValidator { @Value("${config.ordering.field}") private String orderingField; - @Value("${config.success-threshold-check.success-rate}") + @Value("${config.success-threshold-check.success-threshold}") private int successRate; + @Value("${config.success-threshold-check.max-retry}") + private int maxThresholdCheckRetry; + @Value("${config.formatting.standard}") private String standard; + @Autowired + private WorkerConfig workerConfig; + @PostConstruct private void validate() { - validateOrderingConfig(); - validateSuccessThresholdConfig(); - validateFormattingStandard(); + if (workerConfig.isOrderingWorkerEnabled) { + validateOrderingConfig(); + } + if (workerConfig.isSuccessThresholdCheckEnabled) { + validateSuccessThresholdConfig(); + validateMaxRetryFromThresholdCheck(); + } + if (workerConfig.isFormattingWorkerEnabled) { + validateFormattingStandard(); + } + } + + private void validateMaxRetryFromThresholdCheck() { + if (maxThresholdCheckRetry <= 0) { + logger.error("Invalid maxThresholdCheckRetry count set. Needs to be +ve integer"); + throw new ConfigurationValidationException("Invalid maxThresholdCheckRetry count set. Needs to be +ve integer"); + } } // validates the standard to be used for formatting diff --git a/src/main/java/org/mifos/processor/bulk/OperationsAppConfig.java b/src/main/java/org/mifos/processor/bulk/OperationsAppConfig.java index 4ac5c5ac..c738fba6 100644 --- a/src/main/java/org/mifos/processor/bulk/OperationsAppConfig.java +++ b/src/main/java/org/mifos/processor/bulk/OperationsAppConfig.java @@ -14,10 +14,28 @@ public class OperationsAppConfig { @Value("${operations-app.endpoints.batch-transaction}") public String batchTransactionEndpoint; + @Value("${operations-app.endpoints.batch-summary}") + public String batchSummaryEndpoint; + + @Value("${operations-app.endpoints.auth}") + public String authEndpoint; + + @Value("${operations-app.username}") + public String username; + + @Value("${operations-app.password}") + public String password; + public String batchTransactionUrl; + public String batchSummaryUrl; + + public String authUrl; + @PostConstruct private void setup() { batchTransactionUrl = operationAppContactPoint + batchTransactionEndpoint; + batchSummaryUrl = operationAppContactPoint + batchSummaryEndpoint; + authUrl = operationAppContactPoint + authEndpoint; } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelContextConfig.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelContextConfig.java index 348a0747..a3749cb4 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelContextConfig.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelContextConfig.java @@ -1,12 +1,16 @@ package org.mifos.processor.bulk.camel.config; import org.apache.camel.CamelContext; +import org.apache.camel.component.http.HttpComponent; import org.apache.camel.spi.RestConfiguration; import org.apache.camel.spring.boot.CamelContextConfiguration; +import org.apache.camel.support.jsse.SSLContextParameters; +import org.apache.camel.support.jsse.TrustManagersParameters; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import javax.net.ssl.X509ExtendedTrustManager; import java.util.HashMap; @Configuration @@ -15,6 +19,9 @@ public class CamelContextConfig { @Value("${camel.server-port}") private int serverPort; + @Value("${camel.disable-ssl}") + private boolean disableSSL; + @Bean CamelContextConfiguration contextConfiguration() { return new CamelContextConfiguration() { @@ -25,6 +32,12 @@ public void beforeApplicationStart(CamelContext camelContext) { camelContext.setStreamCaching(true); camelContext.disableJMX(); + if (disableSSL) { + HttpComponent httpComponent = camelContext.getComponent("https", HttpComponent.class); + httpComponent.setHttpClientConfigurer(new HttpClientConfigurerTrustAllCACerts()); + } + + RestConfiguration rest = new RestConfiguration(); camelContext.setRestConfiguration(rest); rest.setComponent("undertow"); diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 98fb1891..6456c740 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -31,4 +31,8 @@ private CamelProperties() {} public static final String FILE_2 = "file2"; + public static final String OPS_APP_ACCESS_TOKEN = "opsAppAccessToken"; + + public static final String BATCH_STATUS_FAILED = "batchStatusFailed"; + } diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java b/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java new file mode 100644 index 00000000..79bbba0f --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java @@ -0,0 +1,73 @@ +package org.mifos.processor.bulk.camel.config; + +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import org.apache.camel.component.http.HttpClientConfigurer; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.conn.ssl.TrustStrategy; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class HttpClientConfigurerTrustAllCACerts implements HttpClientConfigurer { + + public Logger logger = LoggerFactory.getLogger(this.getClass()); + + public HttpClientConfigurerTrustAllCACerts() { + } + + @Override + public void configureHttpClient(HttpClientBuilder clientBuilder) { + // setup a Trust Strategy that allows all certificates. + // + SSLContext sslContext = null; + try { + sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() { + public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException { + return true; + } + }).build(); + } catch (KeyManagementException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (NoSuchAlgorithmException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (KeyStoreException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + clientBuilder.setSslcontext( sslContext); + + // don't check Hostnames, either. + // -- use SSLConnectionSocketFactory.getDefaultHostnameVerifier(), if you don't want to weaken + HostnameVerifier hostnameVerifier = SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER; + + // here's the special part: + // -- need to create an SSL Socket Factory, to use our weakened "trust strategy"; + // -- and create a Registry, to register it. + // + SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, hostnameVerifier); + Registry socketFactoryRegistry = RegistryBuilder.create() + .register("http", PlainConnectionSocketFactory.getSocketFactory()) + .register("https", sslSocketFactory) + .build(); + + // now, we create connection-manager using our Registry. + // -- allows multi-threaded use + PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager( socketFactoryRegistry); + clientBuilder.setConnectionManager(connMgr); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java index 0e386213..63c8ed29 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java @@ -10,7 +10,6 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; - import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; @@ -42,7 +41,8 @@ public RouteDefinition getBaseExternalApiRequestRouteDefinition(String routeId, .removeHeader("*") .setHeader(Exchange.HTTP_METHOD, constant(httpMethod.text)) .setHeader("X-Date", simple(ZonedDateTime.now( ZoneOffset.UTC ).format( DateTimeFormatter.ISO_INSTANT ))) - .setHeader("Content-Type", constant("application/json")); + .setHeader("Content-Type", constant("application/json;charset=UTF-8")) + .setHeader("Accept", constant("application/json, text/plain, */*")); } protected enum HttpRequestMethod { diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java new file mode 100644 index 00000000..1904b0a6 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java @@ -0,0 +1,87 @@ +package org.mifos.processor.bulk.camel.routes; + +import org.apache.camel.Exchange; +import org.apache.camel.LoggingLevel; +import org.apache.camel.model.dataformat.JsonLibrary; +import org.mifos.processor.bulk.schema.BatchDTO; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_STATUS_FAILED; +import static org.mifos.processor.bulk.camel.config.CamelProperties.OPS_APP_ACCESS_TOKEN; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; + +@Component +public class BatchStatusRoute extends BaseRouteBuilder { + + @Value("${config.success-threshold-check.success-threshold}") + private int successThreshold; + + @Override + public void configure() throws Exception { + + from("rest:get:test/batch/summary") + .to(RouteId.BATCH_STATUS.getValue()); + + /** + * Base route for kicking off ordering logic. Performs below tasks. + * 1. Downloads the csv form cloud. + * 2. Builds the [Transaction] array using [direct:get-transaction-array] route. + * 3. Format the data based on the configuration provided in application.yaml. @see [Standard.java] + * 4. Update file with the updated data. + * 5. Uploads the updated file in cloud. + */ + from(RouteId.BATCH_STATUS.getValue()) + .id(RouteId.BATCH_STATUS.getValue()) + .log("Starting route " + RouteId.BATCH_STATUS.name()) + .to("direct:get-access-token") + .choice() + .when(exchange -> exchange.getProperty(OPS_APP_ACCESS_TOKEN, String.class) != null) + .log(LoggingLevel.INFO, "Got access token, moving on to API call") + .to("direct:batch-summary") + .to("direct:batch-summary-response-handler") + .otherwise() + .log(LoggingLevel.INFO, "Authentication failed.") + .endChoice(); + + + getBaseExternalApiRequestRouteDefinition("batch-summary", HttpRequestMethod.GET) + .setHeader(Exchange.REST_HTTP_QUERY, simple("batchId=${exchangeProperty." + BATCH_ID + "}")) + .setHeader("Authorization", simple("Bearer ${exchangeProperty."+OPS_APP_ACCESS_TOKEN+"}")) + .setHeader("Platform-TenantId", simple("${exchangeProperty." + TENANT_ID + "}")) + .process(exchange -> { + logger.info(exchange.getIn().getHeaders().toString()); + }) + .toD(operationsAppConfig.batchSummaryUrl + "?bridgeEndpoint=true") + .log(LoggingLevel.INFO, "Batch summary API response: \n\n ${body}"); + + from("direct:batch-summary-response-handler") + .id("direct:batch-summary-response-handler") + .log("Starting route direct:batch-summary-response-handler") + //.setBody(exchange -> exchange.getIn().getBody(String.class)) + .choice() + .when(header("CamelHttpResponseCode").isEqualTo("200")) + .log(LoggingLevel.INFO, "Batch summary request successful") + .unmarshal().json(JsonLibrary.Jackson, BatchDTO.class) + .process(exchange -> { + BatchDTO batchSummary = exchange.getIn().getBody(BatchDTO.class); + + long percentage = (batchSummary.getCompleted()/batchSummary.getTotalTransactions())*100; + + if (percentage >= successThreshold) { + logger.info("Batch success threshold reached. Expected rate: {}, Actual Rate: {}", + successThreshold, percentage); + } + + exchange.setProperty(SUCCESS_RATE, percentage); + + }) + .otherwise() + .log(LoggingLevel.ERROR, "Batch summary request unsuccessful") + .process(exchange -> { + exchange.setProperty(BATCH_STATUS_FAILED, true); + exchange.setProperty(ERROR_DESCRIPTION, exchange.getIn().getBody(String.class)); + exchange.setProperty(ERROR_CODE, exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE)); + }); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/OperationsAuthRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/OperationsAuthRoute.java new file mode 100644 index 00000000..6f31b074 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/OperationsAuthRoute.java @@ -0,0 +1,71 @@ +package org.mifos.processor.bulk.camel.routes; + +import org.apache.camel.Exchange; +import org.apache.camel.LoggingLevel; +import org.apache.camel.model.dataformat.JsonLibrary; +import org.springframework.stereotype.Component; +import java.util.HashMap; +import java.util.Map; +import static org.mifos.processor.bulk.camel.config.CamelProperties.OPS_APP_ACCESS_TOKEN; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; + +@Component +public class OperationsAuthRoute extends BaseRouteBuilder { + + @Override + public void configure() throws Exception { + + from("rest:get:test/auth") + .to("direct:get-access-token"); + + /** + * Error handling route + */ + from("direct:access-token-error") + .id("access-token-error") + .process(exchange -> { + logger.error("Error while fetching Access Token from server: " + exchange.getIn().getBody()); + }); + + /** + * Save Access Token to AccessTokenStore + */ + from("direct:access-token-save") + .id("access-token-save") + .unmarshal().json(JsonLibrary.Jackson, HashMap.class) + .process(exchange -> { + // TODO: Figure out access token storage if required + Map jsonObject = exchange.getIn().getBody(HashMap.class); + exchange.setProperty(OPS_APP_ACCESS_TOKEN, jsonObject.get("access_token")); + logger.debug("Saved Access Token: " + exchange.getProperty(OPS_APP_ACCESS_TOKEN, String.class)); + exchange.getIn().setBody(jsonObject.toString()); + }); + + /** + * Fetch Access Token from SLCB + */ + getBaseExternalApiRequestRouteDefinition("access-token-fetch", HttpRequestMethod.POST) + .setHeader(Exchange.REST_HTTP_QUERY, + simpleF("username=%s&password=%s&grant_type=%s", + operationsAppConfig.username, operationsAppConfig.password, "password")) + .setHeader("Authorization", constant("Basic Y2xpZW50Og==")) + .setHeader("Platform-TenantId", simple("${exchangeProperty." + TENANT_ID + "}")) + .toD(operationsAppConfig.authUrl + "?bridgeEndpoint=true") + .log(LoggingLevel.INFO, "Auth response: \n\n ${body}"); + + + /** + * Access Token check validity and return value + */ + from("direct:get-access-token") + .id("get-access-token") + .to("direct:access-token-fetch") + .choice() + .when(header("CamelHttpResponseCode").isEqualTo("200")) + .log("Access Token Fetch Successful") + .to("direct:access-token-save") + .otherwise() + .log("Access Token Fetch Unsuccessful") + .to("direct:access-token-error"); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 17cbd421..d07653ac 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -1,7 +1,7 @@ package org.mifos.processor.bulk.camel.routes; -import org.apache.camel.attachment.AttachmentMessage; import org.mifos.processor.bulk.file.FileTransferService; +import org.mifos.processor.bulk.utility.Utils; import org.mifos.processor.bulk.zeebe.ZeebeProcessStarter; import org.mifos.processor.bulk.zeebe.worker.WorkerConfig; import org.slf4j.Logger; @@ -12,9 +12,7 @@ import org.springframework.stereotype.Component; import java.io.File; import java.io.FileWriter; -import java.io.IOException; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.UUID; @@ -41,6 +39,15 @@ public class ProcessorStartRoute extends BaseRouteBuilder { @Value("${bpmn.flows.bulk-processor}") private String workflowId; + @Value("${config.success-threshold-check.success-threshold}") + private int successThreshold; + + @Value("${config.success-threshold-check.max-retry}") + private int maxThresholdCheckRetry; + + @Value("${config.success-threshold-check.delay}") + private int thresholdCheckDelay; + private final Logger logger = LoggerFactory.getLogger(this.getClass()); @Override @@ -96,6 +103,9 @@ private void setup() { variables.put(FORMATTING_ENABLED, workerConfig.isFormattingWorkerEnabled); variables.put(SUCCESS_THRESHOLD_CHECK_ENABLED, workerConfig.isSuccessThresholdCheckEnabled); variables.put(MERGE_ENABLED, workerConfig.isMergeBackWorkerEnabled); + variables.put(MAX_STATUS_RETRY, maxThresholdCheckRetry); + variables.put(SUCCESS_THRESHOLD, successThreshold); + variables.put(THRESHOLD_DELAY, Utils.getZeebeTimerValue(thresholdCheckDelay)); zeebeProcessStarter.startZeebeWorkflow(workflowId, "", variables); exchange.getIn().setBody(batchId); diff --git a/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java b/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java new file mode 100644 index 00000000..f970f817 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java @@ -0,0 +1,156 @@ +package org.mifos.processor.bulk.schema; + +import java.math.BigDecimal; +import java.util.Date; + +public class BatchDTO { + + private String batchId; + + private String requestId; + + private Long totalTransactions; + + private Long ongoing; + + private Long failed; + + private Long completed; + + private BigDecimal total_amount; + + private BigDecimal completed_amount; + + private BigDecimal ongoing_amount; + + private BigDecimal failed_amount; + + private String result_file; + + private Date resultGeneratedAt; + + private String note; + + public BatchDTO() { + } + + public BatchDTO(String batchId, String requestId, Long totalTransactions, Long ongoing, Long failed, Long completed, BigDecimal total_amount, BigDecimal completed_amount, BigDecimal ongoing_amount, BigDecimal failed_amount, String result_file, Date resultGeneratedAt, String note) { + this.batchId = batchId; + this.requestId = requestId; + this.totalTransactions = totalTransactions; + this.ongoing = ongoing; + this.failed = failed; + this.completed = completed; + this.total_amount = total_amount; + this.completed_amount = completed_amount; + this.ongoing_amount = ongoing_amount; + this.failed_amount = failed_amount; + this.result_file = result_file; + this.resultGeneratedAt = resultGeneratedAt; + this.note = note; + } + + public String getBatchId() { + return batchId; + } + + public void setBatchId(String batchId) { + this.batchId = batchId; + } + + public String getRequestId() { + return requestId; + } + + public void setRequestId(String requestId) { + this.requestId = requestId; + } + + public Long getTotalTransactions() { + return totalTransactions; + } + + public void setTotalTransactions(Long totalTransactions) { + this.totalTransactions = totalTransactions; + } + + public Long getOngoing() { + return ongoing; + } + + public void setOngoing(Long ongoing) { + this.ongoing = ongoing; + } + + public Long getFailed() { + return failed; + } + + public void setFailed(Long failed) { + this.failed = failed; + } + + public Long getCompleted() { + return completed; + } + + public void setCompleted(Long completed) { + this.completed = completed; + } + + public BigDecimal getTotal_amount() { + return total_amount; + } + + public void setTotal_amount(BigDecimal total_amount) { + this.total_amount = total_amount; + } + + public BigDecimal getCompleted_amount() { + return completed_amount; + } + + public void setCompleted_amount(BigDecimal completed_amount) { + this.completed_amount = completed_amount; + } + + public BigDecimal getOngoing_amount() { + return ongoing_amount; + } + + public void setOngoing_amount(BigDecimal ongoing_amount) { + this.ongoing_amount = ongoing_amount; + } + + public BigDecimal getFailed_amount() { + return failed_amount; + } + + public void setFailed_amount(BigDecimal failed_amount) { + this.failed_amount = failed_amount; + } + + public String getResult_file() { + return result_file; + } + + public void setResult_file(String result_file) { + this.result_file = result_file; + } + + public Date getResultGeneratedAt() { + return resultGeneratedAt; + } + + public void setResultGeneratedAt(Date resultGeneratedAt) { + this.resultGeneratedAt = resultGeneratedAt; + } + + public String getNote() { + return note; + } + + public void setNote(String note) { + this.note = note; + } +} diff --git a/src/main/java/org/mifos/processor/bulk/utility/Utils.java b/src/main/java/org/mifos/processor/bulk/utility/Utils.java index 5438ea95..cd2de7be 100644 --- a/src/main/java/org/mifos/processor/bulk/utility/Utils.java +++ b/src/main/java/org/mifos/processor/bulk/utility/Utils.java @@ -1,6 +1,12 @@ package org.mifos.processor.bulk.utility; +import javax.net.ssl.*; import java.io.*; +import java.net.URL; +import java.net.URLConnection; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.X509Certificate; public class Utils { @@ -36,4 +42,26 @@ public static String getAwsFileUrl(String baseUrl, String filename) { return String.format("%s/%s", baseUrl, filename); } + /** + * takes initial timer in the ISO 8601 durations format + * for more info check + * https://docs.camunda.io/docs/0.26/reference/bpmn-workflows/timer-events/#time-duration + * + * @param initialTimer initial timer in the ISO 8601 durations format, ex: PT45S + * @return next timer value in the ISO 8601 durations format + */ + public static String getNextTimer(String initialTimer){ + String stringSecondsValue = initialTimer.split("T")[1].split("S")[0]; + int initialSeconds = Integer.parseInt(stringSecondsValue); + + int currentPower = (int) ( Math.log(initialSeconds) / Math.log(2) ); + int next = (int) Math.pow(2, ++currentPower); + + return String.format("PT%sS", next); + } + + public static String getZeebeTimerValue(int timer) { + return String.format("PT%sS", timer); + } + } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 6425d4b8..6bbb7001 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -82,4 +82,18 @@ private ZeebeVariables() { public static final String MERGE_COMPLETED = "mergeCompleted"; public static final String RESULT_FILE = "resultFile"; + + public static final String MAX_STATUS_RETRY = "maxStatusRetry"; + + public static final String RETRY = "retry"; + + public static final String SUCCESS_THRESHOLD = "successThreshold"; + + public static final String SUCCESS_RATE = "successRate"; + + public static final String ERROR_CODE = "errorCode"; + + public static final String ERROR_DESCRIPTION = "errorDescription"; + + public static final String THRESHOLD_DELAY = "thresholdDelay"; } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java index e3113f16..4945de5c 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java @@ -1,11 +1,44 @@ package org.mifos.processor.bulk.zeebe.worker; +import org.apache.camel.Exchange; +import org.apache.camel.support.DefaultExchange; +import org.mifos.processor.bulk.camel.routes.RouteId; +import org.springframework.stereotype.Component; +import java.util.Map; +import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_STATUS_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; + +@Component public class BatchStatusWorker extends BaseWorker { @Override public void setup() { newWorker(Worker.BATCH_STATUS, (client, job) -> { - client.newCompleteCommand(job.getKey()).send(); + Map variables = job.getVariablesAsMap(); + + int retry = (int) variables.getOrDefault(RETRY, 0); + int successRate = (int) variables.getOrDefault(SUCCESS_RATE, 0); + + Exchange exchange = new DefaultExchange(camelContext); + exchange.setProperty(BATCH_ID, variables.get(BATCH_ID)); + exchange.setProperty(TENANT_ID, variables.get(TENANT_ID)); + + sendToCamelRoute(RouteId.BATCH_STATUS, exchange); + + Boolean batchStatusFailed = exchange.getProperty(BATCH_STATUS_FAILED, Boolean.class); + if (batchStatusFailed == null || !batchStatusFailed) { + successRate = exchange.getProperty(SUCCESS_RATE, Long.class).intValue(); + } else { + variables.put(ERROR_CODE, exchange.getProperty(ERROR_CODE)); + variables.put(ERROR_DESCRIPTION, exchange.getProperty(ERROR_DESCRIPTION)); + logger.info("Error: {}, {}", variables.get(ERROR_CODE), variables.get(ERROR_DESCRIPTION)); + } + + variables.put(SUCCESS_RATE, successRate); + variables.put(RETRY, ++retry); + + logger.info("Retry: {} and Success Rate: {}", retry, successRate); + client.newCompleteCommand(job.getKey()).variables(variables).send(); }); } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index dfa92f32..41b2ebbe 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -1,5 +1,6 @@ camel: server-port: 5000 + disable-ssl: false springboot: main-run-controller: true dataformat: @@ -21,8 +22,12 @@ zeebe: contactpoint: "zeebe-zeebe-gateway:26500" operations-app: - contactpoint: "https://ops-bk.ibank.financial" + contactpoint: "https://ops-bk.sandbox.fynarfin.io" + username: "mifos" + password: "password" endpoints: + auth: "/oauth/token" + batch-summary: "/api/v1/batch" batch-transaction: "/api/v1/batch/transactions" cloud: @@ -81,4 +86,6 @@ config: enable: false success-threshold-check: enable: false - success-rate: 95 # in percentage + success-threshold: 95 # in percentage + max-retry: 4 + delay: 60 # in seconds From 6aea93c642d487082404791c8d9cefe015a8ed4f Mon Sep 17 00:00:00 2001 From: DanishJamal Date: Fri, 9 Sep 2022 05:46:14 +0530 Subject: [PATCH 029/156] Minor code optimisation --- .../org/mifos/processor/BulkProcessorApplication.java | 6 ++++++ .../processor/bulk/camel/config/CamelContextConfig.java | 6 +++++- .../config/HttpClientConfigurerTrustAllCACerts.java | 9 +-------- .../java/org/mifos/processor/bulk/utility/Utils.java | 8 +++++--- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/mifos/processor/BulkProcessorApplication.java b/src/main/java/org/mifos/processor/BulkProcessorApplication.java index c523eee4..280a85f1 100644 --- a/src/main/java/org/mifos/processor/BulkProcessorApplication.java +++ b/src/main/java/org/mifos/processor/BulkProcessorApplication.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import org.apache.camel.Processor; +import org.mifos.processor.bulk.camel.config.HttpClientConfigurerTrustAllCACerts; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.Bean; @@ -39,4 +40,9 @@ public CsvMapper csvMapper() { return new CsvMapper(); } + @Bean + public HttpClientConfigurerTrustAllCACerts httpClientConfigurer() { + return new HttpClientConfigurerTrustAllCACerts(); + } + } diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelContextConfig.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelContextConfig.java index a3749cb4..a7a88538 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelContextConfig.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelContextConfig.java @@ -6,6 +6,7 @@ import org.apache.camel.spring.boot.CamelContextConfiguration; import org.apache.camel.support.jsse.SSLContextParameters; import org.apache.camel.support.jsse.TrustManagersParameters; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -22,6 +23,9 @@ public class CamelContextConfig { @Value("${camel.disable-ssl}") private boolean disableSSL; + @Autowired + private HttpClientConfigurerTrustAllCACerts httpClientConfigurerTrustAllCACerts; + @Bean CamelContextConfiguration contextConfiguration() { return new CamelContextConfiguration() { @@ -34,7 +38,7 @@ public void beforeApplicationStart(CamelContext camelContext) { if (disableSSL) { HttpComponent httpComponent = camelContext.getComponent("https", HttpComponent.class); - httpComponent.setHttpClientConfigurer(new HttpClientConfigurerTrustAllCACerts()); + httpComponent.setHttpClientConfigurer(httpClientConfigurerTrustAllCACerts); } diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java b/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java index 79bbba0f..d87fd5bb 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java @@ -38,14 +38,7 @@ public boolean isTrusted(X509Certificate[] arg0, String arg1) throws Certificate return true; } }).build(); - } catch (KeyManagementException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (NoSuchAlgorithmException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (KeyStoreException e) { - // TODO Auto-generated catch block + } catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) { e.printStackTrace(); } clientBuilder.setSslcontext( sslContext); diff --git a/src/main/java/org/mifos/processor/bulk/utility/Utils.java b/src/main/java/org/mifos/processor/bulk/utility/Utils.java index cd2de7be..126c72d2 100644 --- a/src/main/java/org/mifos/processor/bulk/utility/Utils.java +++ b/src/main/java/org/mifos/processor/bulk/utility/Utils.java @@ -21,10 +21,11 @@ public static String mergeCsvFile(String file1, String file2) { // create a reader for tmpFile BufferedReader in = new BufferedReader(new FileReader(file2)); String str; - int i = 0; + boolean isFirstLine = true; while ((str = in.readLine()) != null) { - if (i == 0) { - i = 1; + if (isFirstLine) { + // used for skipping header writing + isFirstLine = false; continue; } out.write(str+"\n"); @@ -32,6 +33,7 @@ public static String mergeCsvFile(String file1, String file2) { in.close(); out.close(); } catch (IOException e) { + e.printStackTrace(); return null; } From 349b2c105f4c98555e541a55a0e113b0e29494be Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Wed, 14 Sep 2022 20:19:02 +0530 Subject: [PATCH 030/156] Batch transactions API updated as per specs (#21) * Batch transactions API updated as per specs * Added new zeebe variable for payment mode --- .../bulk/camel/config/CamelProperties.java | 2 + .../bulk/camel/routes/InitSubBatchRoute.java | 1 + .../camel/routes/ProcessorStartRoute.java | 140 ++++++++++++++---- .../processor/bulk/zeebe/ZeebeVariables.java | 2 + 4 files changed, 120 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 6456c740..c498a04c 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -35,4 +35,6 @@ private CamelProperties() {} public static final String BATCH_STATUS_FAILED = "batchStatusFailed"; + public static final String BATCH_REQUEST_TYPE = "batchRequestType"; + } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index aa094b1f..d2cf0ecf 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -58,6 +58,7 @@ public void configure() throws Exception { variables.put(ONGOING_AMOUNT, exchange.getProperty(ONGOING_AMOUNT)); variables.put(FAILED_AMOUNT, exchange.getProperty(FAILED_AMOUNT)); variables.put(COMPLETED_AMOUNT, exchange.getProperty(COMPLETED_AMOUNT)); + variables.put(PAYMENT_MODE, "slcb"); zeebeProcessStarter.startZeebeWorkflow( Utils.getTenantSpecificWorkflowId(bpmnConfig.slcbBpmn, tenantName), variables); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index d07653ac..cdf98198 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -1,5 +1,8 @@ package org.mifos.processor.bulk.camel.routes; +import org.apache.camel.Exchange; +import org.apache.camel.LoggingLevel; +import org.json.JSONObject; import org.mifos.processor.bulk.file.FileTransferService; import org.mifos.processor.bulk.utility.Utils; import org.mifos.processor.bulk.zeebe.ZeebeProcessStarter; @@ -16,6 +19,7 @@ import java.util.Map; import java.util.UUID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_REQUEST_TYPE; import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @@ -56,6 +60,32 @@ public void configure() { } private void setup() { + + from("rest:POST:/batchtransactions") + .id("rest:POST:/batchtransactions") + .log("Starting route rest:POST:/batchtransactions") + .to("direct:validate-tenant") + .process(exchange -> { + String filename = exchange.getIn().getHeader("filename", String.class); + String requestId = exchange.getIn().getHeader("X-CorrelationID", String.class); + String purpose = exchange.getIn().getHeader("Purpose", String.class); + String type = exchange.getIn().getHeader("Type", String.class); + exchange.setProperty(FILE_NAME, filename); + exchange.setProperty(REQUEST_ID, requestId); + exchange.setProperty(PURPOSE, purpose); + exchange.setProperty(BATCH_REQUEST_TYPE, type); + }) + .choice() + .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("raw")) + .to("direct:start-batch-process-raw") + .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("csv")) + .unmarshal().mimeMultipart("multipart/*") + .to("direct:start-batch-process-csv") + .otherwise() + .setBody(exchange -> getUnsupportedTypeJson(exchange.getProperty(BATCH_REQUEST_TYPE, String.class)).toString()) + .log("Completed execution of route rest:POST:/batchtransactions"); + + from("rest:POST:/bulk/transfer/{requestId}/{fileName}") .unmarshal().mimeMultipart("multipart/*") .to("direct:validate-tenant") @@ -63,8 +93,36 @@ private void setup() { String fileName = System.currentTimeMillis() + "_" + exchange.getIn().getHeader("fileName", String.class); String requestId = exchange.getIn().getHeader("requestId", String.class); String purpose = exchange.getIn().getHeader("purpose", String.class); + + exchange.setProperty(FILE_NAME, fileName); + exchange.setProperty(REQUEST_ID, requestId); + exchange.setProperty(PURPOSE, purpose); + }) + .to("direct:start-batch-process-csv"); + + from("direct:validate-tenant") + .id("direct:validate-tenant") + .log("Validating tenant") + .process(exchange -> { + String tenantName = exchange.getIn().getHeader("Platform-TenantId", String.class); + if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { + throw new Exception("Invalid tenant value."); + } + exchange.setProperty(TENANT_NAME, tenantName); + }) + .setHeader("Content-Type", constant("application/json;charset=UTF-8")) + .log("Completed route direct:validate-tenant"); + + from("direct:start-batch-process-csv") + .id("direct:start-batch-process-csv") + .log("Starting route direct:start-batch-process-csv") + .process(exchange -> { + String fileName = exchange.getProperty(FILE_NAME, String.class); + String requestId = exchange.getProperty(REQUEST_ID, String.class); + String purpose = exchange.getProperty(PURPOSE, String.class); String batchId = UUID.randomUUID().toString(); + if (purpose == null || purpose.isEmpty()) { purpose = "test payment"; } @@ -81,7 +139,6 @@ private void setup() { fileWriter.write(csvData); fileWriter.close(); - logger.info(csvData); logger.info(""+file.length()); logger.info(file.getAbsolutePath()); @@ -89,39 +146,72 @@ private void setup() { logger.info("File uploaded {}", nm); - + Map variables = new HashMap<>(); variables.put(BATCH_ID, batchId); variables.put(FILE_NAME, fileName); variables.put(REQUEST_ID, requestId); variables.put(PURPOSE, purpose); variables.put(TENANT_ID, exchange.getProperty(TENANT_NAME)); - variables.put(PARTY_LOOKUP_ENABLED, workerConfig.isPartyLookUpWorkerEnabled); - variables.put(APPROVAL_ENABLED, workerConfig.isApprovalWorkerEnabled); - variables.put(ORDERING_ENABLED, workerConfig.isOrderingWorkerEnabled); - variables.put(SPLITTING_ENABLED, workerConfig.isSplittingWorkerEnabled); - variables.put(FORMATTING_ENABLED, workerConfig.isFormattingWorkerEnabled); - variables.put(SUCCESS_THRESHOLD_CHECK_ENABLED, workerConfig.isSuccessThresholdCheckEnabled); - variables.put(MERGE_ENABLED, workerConfig.isMergeBackWorkerEnabled); - variables.put(MAX_STATUS_RETRY, maxThresholdCheckRetry); - variables.put(SUCCESS_THRESHOLD, successThreshold); - variables.put(THRESHOLD_DELAY, Utils.getZeebeTimerValue(thresholdCheckDelay)); - - zeebeProcessStarter.startZeebeWorkflow(workflowId, "", variables); - exchange.getIn().setBody(batchId); - }); + setConfigProperties(variables); + + JSONObject response = new JSONObject(); + + try { + String txnId = zeebeProcessStarter.startZeebeWorkflow(workflowId, "", variables); + if (txnId == null || txnId.isEmpty()) { + response.put("errorCode", 500); + response.put("errorDescription", "Unable to start zeebe workflow"); + response.put("developerMessage", "Issue in starting the zeebe workflow, check the zeebe configuration"); + } else { + response.put("batch_id", batchId); + response.put("request_id", requestId); + response.put("status", "queued"); + } + } catch (Exception e) { + response.put("errorCode", 500); + response.put("errorDescription", "Unable to start zeebe workflow"); + response.put("developerMessage", e.getLocalizedMessage()); + } - from("direct:validate-tenant") - .id("direct:validate-tenant") - .log("Validating tenant") + exchange.getIn().setBody(response.toString()); + + }) + .log("Completed route direct:start-batch-process-csv"); + + + from("direct:start-batch-process-raw") + .id("direct:start-batch-process-raw") + .log("Starting route direct:start-batch-process-raw") .process(exchange -> { - String tenantName = exchange.getIn().getHeader("Platform-TenantId", String.class); - if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { - throw new Exception("Invalid tenant value."); - } - exchange.setProperty(TENANT_NAME, tenantName); - }); + JSONObject response = new JSONObject(); + response.put("batch_id", UUID.randomUUID().toString()); + response.put("request_id", UUID.randomUUID().toString()); + response.put("status", "queued"); + exchange.getIn().setBody(response.toString()); + }) + .log("Completed route direct:start-batch-process-raw"); + } + private Map setConfigProperties(Map variables) { + variables.put(PARTY_LOOKUP_ENABLED, workerConfig.isPartyLookUpWorkerEnabled); + variables.put(APPROVAL_ENABLED, workerConfig.isApprovalWorkerEnabled); + variables.put(ORDERING_ENABLED, workerConfig.isOrderingWorkerEnabled); + variables.put(SPLITTING_ENABLED, workerConfig.isSplittingWorkerEnabled); + variables.put(FORMATTING_ENABLED, workerConfig.isFormattingWorkerEnabled); + variables.put(SUCCESS_THRESHOLD_CHECK_ENABLED, workerConfig.isSuccessThresholdCheckEnabled); + variables.put(MERGE_ENABLED, workerConfig.isMergeBackWorkerEnabled); + variables.put(MAX_STATUS_RETRY, maxThresholdCheckRetry); + variables.put(SUCCESS_THRESHOLD, successThreshold); + variables.put(THRESHOLD_DELAY, Utils.getZeebeTimerValue(thresholdCheckDelay)); + return variables; + } + private JSONObject getUnsupportedTypeJson(String type) { + JSONObject response = new JSONObject(); + response.put("errorCode", 400); + response.put("errorDescription", String.format("Query parameter ?type=%s not supported", type)); + response.put("developerMessage", ""); + return response; } } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 6bbb7001..65156399 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -96,4 +96,6 @@ private ZeebeVariables() { public static final String ERROR_DESCRIPTION = "errorDescription"; public static final String THRESHOLD_DELAY = "thresholdDelay"; + + public static final String PAYMENT_MODE = "paymentMode"; } From 000b6b521598694a291dda0dde4f0e18197a9d3a Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Wed, 14 Sep 2022 21:01:50 +0530 Subject: [PATCH 031/156] Transaction schema updated maintaining the backward compatibility (#22) --- .../camel/routes/FileProcessingRoute.java | 49 +++--- .../bulk/camel/routes/InitSubBatchRoute.java | 2 +- .../bulk/camel/routes/OrderingRoute.java | 6 +- .../bulk/format/helper/GSMAMapper.java | 4 +- .../processor/bulk/schema/TestSchema.java | 149 ++++++++++++++++++ .../processor/bulk/schema/Transaction.java | 128 ++++++++++++--- 6 files changed, 285 insertions(+), 53 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/schema/TestSchema.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java index ebaafc2d..6be82fea 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java @@ -1,15 +1,13 @@ package org.mifos.processor.bulk.camel.routes; import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.databind.SequenceWriter; import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.dataformat.csv.CsvSchema; import org.mifos.processor.bulk.schema.Transaction; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; +import java.io.*; import java.util.ArrayList; import java.util.List; import static org.mifos.processor.bulk.camel.config.CamelProperties.*; @@ -64,34 +62,37 @@ public void configure() { from("direct:update-file") .id("direct:update-file") .log("Starting route direct:update-file") + .to("direct:update-file-v2") + .log("Update complete"); + + /** + * this is backward compatible version of update-file route for new CSV schema + * exchangeInput: + * [LOCAL_FILE_PATH] the absolute path to the csv file + * [TRANSACTION_LIST] containing the list of [Transaction] + * [OVERRIDE_HEADER] if set to true will override the header or else use the existing once in csv file + */ + from("direct:update-file-v2") + .id("direct:update-file-v2") + .log("Starting route direct:update-file-v2") .process(exchange -> { String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); - List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); // getting header Boolean overrideHeader = exchange.getProperty(OVERRIDE_HEADER, Boolean.class); - String header; - if (overrideHeader == null || !overrideHeader) { - BufferedReader reader = new BufferedReader(new FileReader(filepath)); - header = reader.readLine() + System.lineSeparator(); - reader.close(); + CsvSchema csvSchema = csvMapper.schemaFor(Transaction.class); + if (overrideHeader) { + csvSchema = csvSchema.withHeader(); } else { - header = transactionList.get(0).getCsvHeader() + System.lineSeparator(); + csvSchema = csvSchema.withoutHeader(); } - FileWriter writer = new FileWriter(filepath); - writer.write(header); - - // updating file with re-ordered data - transactionList.forEach(transaction -> { - try { - writer.write(transaction.getCsvString() + System.lineSeparator()); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - writer.close(); - logger.info("Update complete"); + File file = new File(filepath); + SequenceWriter writer = csvMapper.writerWithSchemaFor(Transaction.class).with(csvSchema).writeValues(file); + for (Transaction transaction: transactionList) { + writer.write(transaction); + } }); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index d2cf0ecf..90eb3f7a 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -47,7 +47,7 @@ public void configure() throws Exception { .process(exchange -> { String tenantName = exchange.getProperty(TENANT_NAME, String.class); List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); - if (transactionList.get(0).getPayment_mode().equalsIgnoreCase("slcb")) { + if (transactionList.get(0).getPaymentMode().equalsIgnoreCase("slcb")) { Map variables = new HashMap<>(); variables.put(BATCH_ID, exchange.getProperty(BATCH_ID)); variables.put(SUB_BATCH_ID, UUID.randomUUID().toString()); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java index 2767880d..d5261416 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java @@ -55,10 +55,10 @@ public void configure() { key = "" + transaction.getId(); break; case "request_id": - key = transaction.getRequest_id(); + key = transaction.getRequestId(); break; case "account_number": - key = transaction.getAccount_number(); + key = transaction.getAccountNumber(); break; case "amount": key = transaction.getAmount(); @@ -70,7 +70,7 @@ public void configure() { key = transaction.getNote(); break; default: - key = transaction.getPayment_mode(); + key = transaction.getPaymentMode(); break; } diff --git a/src/main/java/org/mifos/processor/bulk/format/helper/GSMAMapper.java b/src/main/java/org/mifos/processor/bulk/format/helper/GSMAMapper.java index 4903a239..ca924cf5 100644 --- a/src/main/java/org/mifos/processor/bulk/format/helper/GSMAMapper.java +++ b/src/main/java/org/mifos/processor/bulk/format/helper/GSMAMapper.java @@ -11,8 +11,8 @@ public GSMATransaction convert(Transaction object) { GSMATransaction gsmaTransaction = new GSMATransaction(); gsmaTransaction.setId(object.getId()); gsmaTransaction.setRequestId("test"); - gsmaTransaction.setPaymentMode(object.getPayment_mode()); - gsmaTransaction.setAccountNumber(object.getAccount_number()); + gsmaTransaction.setPaymentMode(object.getPaymentMode()); + gsmaTransaction.setAccountNumber(object.getAccountNumber()); gsmaTransaction.setAmount(object.getAmount()); gsmaTransaction.setNote(object.getNote()); return gsmaTransaction; diff --git a/src/main/java/org/mifos/processor/bulk/schema/TestSchema.java b/src/main/java/org/mifos/processor/bulk/schema/TestSchema.java new file mode 100644 index 00000000..8012724a --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/TestSchema.java @@ -0,0 +1,149 @@ +package org.mifos.processor.bulk.schema; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +// id,request_id,payment_mode,payer_identifier_type, +// payer_identifier,payee_identifier_type,payee_identifier, +// amount,currency,note + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonPropertyOrder({ "id", "request_id", "payment_mode", "payer_identifier_type", "payer_identifier", "payee_identifier_type", "payee_identifier", "currency","amount", "note", "program_shortcode", }) +public class TestSchema { + + @JsonProperty("id") + private int id; + + @JsonProperty("request_id") + private String requestId; + + @JsonProperty("payment_mode") + private String paymentMode; + + @JsonProperty("payer_identifier_type") + private String payerIdentifierType; + + @JsonProperty("payer_identifier") + private String payerIdentifier; + + @JsonProperty("payee_identifier_type") + private String payeeIdentifierType; + + @JsonProperty("payee_identifier") + private String payeeIdentifier; + + @JsonProperty("currency") + private String currency; + + @JsonProperty("amount") + private String amount; + + @JsonProperty("note") + private String note; + + @JsonProperty("program_shortcode") + private String programShortCode; + + @JsonProperty("cycle") + private String cycle; + + public TestSchema() { + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getRequestId() { + return requestId; + } + + public void setRequestId(String requestId) { + this.requestId = requestId; + } + + public String getPaymentMode() { + return paymentMode; + } + + public void setPaymentMode(String paymentMode) { + this.paymentMode = paymentMode; + } + + public String getPayerIdentifierType() { + return payerIdentifierType; + } + + public void setPayerIdentifierType(String payerIdentifierType) { + this.payerIdentifierType = payerIdentifierType; + } + + public String getPayerIdentifier() { + return payerIdentifier; + } + + public void setPayerIdentifier(String payerIdentifier) { + this.payerIdentifier = payerIdentifier; + } + + public String getPayeeIdentifierType() { + return payeeIdentifierType; + } + + public void setPayeeIdentifierType(String payeeIdentifierType) { + this.payeeIdentifierType = payeeIdentifierType; + } + + public String getPayeeIdentifier() { + return payeeIdentifier; + } + + public void setPayeeIdentifier(String payeeIdentifier) { + this.payeeIdentifier = payeeIdentifier; + } + + public String getAmount() { + return amount; + } + + public void setAmount(String amount) { + this.amount = amount; + } + + public String getCurrency() { + return currency; + } + + public void setCurrency(String currency) { + this.currency = currency; + } + + public String getNote() { + return note; + } + + public void setNote(String note) { + this.note = note; + } + + public String getProgramShortCode() { + return programShortCode; + } + + public void setProgramShortCode(String programShortCode) { + this.programShortCode = programShortCode; + } + + public String getCycle() { + return cycle; + } + + public void setCycle(String cycle) { + this.cycle = cycle; + } +} diff --git a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java index b2f76848..27a79f17 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java +++ b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java @@ -1,20 +1,52 @@ package org.mifos.processor.bulk.schema; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.*; +import jdk.nashorn.internal.ir.annotations.Ignore; -@JsonPropertyOrder({ "id", "request_id", "payment_mode", "account_number", "amount", "currency", "note" }) +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonPropertyOrder({ "id", "request_id", "payment_mode", "account_number", "payer_identifier_type", "payer_identifier", "payee_identifier_type", "payee_identifier", "amount", "currency", "note", "program_shortcode", "cycle" }) public class Transaction implements CsvSchema { + @JsonProperty("id") private int id; - private String request_id; - private String payment_mode; - private String account_number; + + @JsonProperty("request_id") + private String requestId; + + @JsonProperty("payment_mode") + private String paymentMode; + + @JsonProperty("account_number") + private String accountNumber; + + @JsonProperty("amount") private String amount; + + @JsonProperty("currency") private String currency; + @JsonProperty("note") private String note; - private String batchId; + @JsonProperty(value = "payer_identifier_type") + private String payerIdentifierType; + + @JsonProperty("payer_identifier") + private String payerIdentifier; + + @JsonProperty("payee_identifier_type") + private String payeeIdentifierType; + + @JsonProperty("payee_identifier") + private String payeeIdentifier; + + @JsonProperty("program_shortcode") + private String programShortCode; + + @JsonProperty("cycle") + private String cycle; + + @JsonIgnore + private String batchId; public int getId() { return id; @@ -24,28 +56,28 @@ public void setId(int id) { this.id = id; } - public String getRequest_id() { - return request_id; + public String getRequestId() { + return requestId; } - public void setRequest_id(String request_id) { - this.request_id = request_id; + public void setRequestId(String requestId) { + this.requestId = requestId; } - public String getPayment_mode() { - return payment_mode; + public String getPaymentMode() { + return paymentMode; } - public void setPayment_mode(String payment_mode) { - this.payment_mode = payment_mode; + public void setPaymentMode(String paymentMode) { + this.paymentMode = paymentMode; } - public String getAccount_number() { - return account_number; + public String getAccountNumber() { + return accountNumber; } - public void setAccount_number(String account_number) { - this.account_number = account_number; + public void setAccountNumber(String accountNumber) { + this.accountNumber = accountNumber; } public String getAmount() { @@ -72,10 +104,12 @@ public void setNote(String note) { this.note = note; } + @JsonIgnore public String getBatchId() { return batchId; } + @JsonIgnore public void setBatchId(String batchId) { this.batchId = batchId; } @@ -84,9 +118,9 @@ public void setBatchId(String batchId) { public String toString() { return "Transaction{" + "id=" + id + - ", request_id='" + request_id + '\'' + - ", payment_mode='" + payment_mode + '\'' + - ", account_number='" + account_number + '\'' + + ", request_id='" + requestId + '\'' + + ", payment_mode='" + paymentMode + '\'' + + ", account_number='" + accountNumber + '\'' + ", amount='" + amount + '\'' + ", currency='" + currency + '\'' + ", note='" + note + '\'' + @@ -94,10 +128,58 @@ public String toString() { '}'; } + public String getPayerIdentifierType() { + return payerIdentifierType; + } + + public void setPayerIdentifierType(String payerIdentifierType) { + this.payerIdentifierType = payerIdentifierType; + } + + public String getPayerIdentifier() { + return payerIdentifier; + } + + public void setPayerIdentifier(String payerIdentifier) { + this.payerIdentifier = payerIdentifier; + } + + public String getPayeeIdentifierType() { + return payeeIdentifierType; + } + + public void setPayeeIdentifierType(String payeeIdentifierType) { + this.payeeIdentifierType = payeeIdentifierType; + } + + public String getPayeeIdentifier() { + return payeeIdentifier; + } + + public void setPayeeIdentifier(String payeeIdentifier) { + this.payeeIdentifier = payeeIdentifier; + } + + public String getProgramShortCode() { + return programShortCode; + } + + public void setProgramShortCode(String programShortCode) { + this.programShortCode = programShortCode; + } + + public String getCycle() { + return cycle; + } + + public void setCycle(String cycle) { + this.cycle = cycle; + } + @JsonIgnore @Override public String getCsvString() { - return String.format("%s,%s,%s,%s,%s,%s,%s", id, request_id, payment_mode, account_number, amount, currency, note); + return String.format("%s,%s,%s,%s,%s,%s,%s", id, requestId, paymentMode, accountNumber, amount, currency, note); } @JsonIgnore From b09c2d134ac4c68140435b7aa2d4c17394991997 Mon Sep 17 00:00:00 2001 From: DanishJamal Date: Fri, 16 Sep 2022 18:51:30 +0530 Subject: [PATCH 032/156] Tenant validation disabled --- .../processor/bulk/camel/routes/ProcessorStartRoute.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index cdf98198..0cdb3304 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -105,9 +105,9 @@ private void setup() { .log("Validating tenant") .process(exchange -> { String tenantName = exchange.getIn().getHeader("Platform-TenantId", String.class); - if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { + /*if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { throw new Exception("Invalid tenant value."); - } + }*/ exchange.setProperty(TENANT_NAME, tenantName); }) .setHeader("Content-Type", constant("application/json;charset=UTF-8")) From d0a1fd8af66c8a9dd2bb0af482bfa3e893e498c4 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Fri, 16 Sep 2022 19:32:12 +0530 Subject: [PATCH 033/156] Update the BatchDTO as per the api spec (#23) * Minor fix in ops-app response handler * Added result file for bulk connector --- .../bulk/camel/routes/BatchStatusRoute.java | 2 +- .../bulk/camel/routes/InitSubBatchRoute.java | 2 + .../mifos/processor/bulk/schema/BatchDTO.java | 182 +++++++++++------- .../bulk/zeebe/worker/MergeBackWorker.java | 5 + 4 files changed, 123 insertions(+), 68 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java index 1904b0a6..9d170d26 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java @@ -65,7 +65,7 @@ public void configure() throws Exception { .process(exchange -> { BatchDTO batchSummary = exchange.getIn().getBody(BatchDTO.class); - long percentage = (batchSummary.getCompleted()/batchSummary.getTotalTransactions())*100; + long percentage = (batchSummary.getSuccessful()/batchSummary.getTotal())*100; if (percentage >= successThreshold) { logger.info("Batch success threshold reached. Expected rate: {}, Actual Rate: {}", diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index 90eb3f7a..d41afec4 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -58,6 +58,8 @@ public void configure() throws Exception { variables.put(ONGOING_AMOUNT, exchange.getProperty(ONGOING_AMOUNT)); variables.put(FAILED_AMOUNT, exchange.getProperty(FAILED_AMOUNT)); variables.put(COMPLETED_AMOUNT, exchange.getProperty(COMPLETED_AMOUNT)); + variables.put(RESULT_FILE, String.format("Result_%s", + exchange.getProperty(SERVER_FILE_NAME))); variables.put(PAYMENT_MODE, "slcb"); zeebeProcessStarter.startZeebeWorkflow( diff --git a/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java b/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java index f970f817..33479aa0 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java +++ b/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java @@ -5,73 +5,97 @@ public class BatchDTO { - private String batchId; + private String batch_id; - private String requestId; + private String request_id; - private Long totalTransactions; + private Long total; private Long ongoing; private Long failed; - private Long completed; + private Long successful; - private BigDecimal total_amount; + private BigDecimal totalAmount; - private BigDecimal completed_amount; + private BigDecimal successfulAmount; - private BigDecimal ongoing_amount; + private BigDecimal pendingAmount; - private BigDecimal failed_amount; + private BigDecimal failedAmount; - private String result_file; + private String file; - private Date resultGeneratedAt; + private String notes; - private String note; + private String created_at; + + private String status; + + private String modes; + + private String purpose; public BatchDTO() { } - public BatchDTO(String batchId, String requestId, Long totalTransactions, Long ongoing, Long failed, Long completed, BigDecimal total_amount, BigDecimal completed_amount, BigDecimal ongoing_amount, BigDecimal failed_amount, String result_file, Date resultGeneratedAt, String note) { - this.batchId = batchId; - this.requestId = requestId; - this.totalTransactions = totalTransactions; + public BatchDTO(String batchId, String requestId, Long totalTransactions, Long ongoing, Long failed, Long completed, BigDecimal total_amount, BigDecimal completed_amount, BigDecimal ongoing_amount, BigDecimal failed_amount, String result_file, String note) { + this.batch_id = batchId; + this.request_id = requestId; + this.total = totalTransactions; + this.ongoing = ongoing; + this.failed = failed; + this.successful = completed; + this.totalAmount = total_amount; + this.successfulAmount = completed_amount; + this.pendingAmount = ongoing_amount; + this.failedAmount = failed_amount; + this.file = result_file; + this.notes = note; + } + + public BatchDTO(String batch_id, String request_id, Long total, Long ongoing, Long failed, Long successful, BigDecimal totalAmount, BigDecimal successfulAmount, BigDecimal pendingAmount, BigDecimal failedAmount, String file, String notes, String created_at, String status, String modes, String purpose) { + this.batch_id = batch_id; + this.request_id = request_id; + this.total = total; this.ongoing = ongoing; this.failed = failed; - this.completed = completed; - this.total_amount = total_amount; - this.completed_amount = completed_amount; - this.ongoing_amount = ongoing_amount; - this.failed_amount = failed_amount; - this.result_file = result_file; - this.resultGeneratedAt = resultGeneratedAt; - this.note = note; + this.successful = successful; + this.totalAmount = totalAmount; + this.successfulAmount = successfulAmount; + this.pendingAmount = pendingAmount; + this.failedAmount = failedAmount; + this.file = file; + this.notes = notes; + this.created_at = created_at; + this.status = status; + this.modes = modes; + this.purpose = purpose; } - public String getBatchId() { - return batchId; + public String getBatch_id() { + return batch_id; } - public void setBatchId(String batchId) { - this.batchId = batchId; + public void setBatch_id(String batch_id) { + this.batch_id = batch_id; } - public String getRequestId() { - return requestId; + public String getRequest_id() { + return request_id; } - public void setRequestId(String requestId) { - this.requestId = requestId; + public void setRequest_id(String request_id) { + this.request_id = request_id; } - public Long getTotalTransactions() { - return totalTransactions; + public Long getTotal() { + return total; } - public void setTotalTransactions(Long totalTransactions) { - this.totalTransactions = totalTransactions; + public void setTotal(Long total) { + this.total = total; } public Long getOngoing() { @@ -90,67 +114,91 @@ public void setFailed(Long failed) { this.failed = failed; } - public Long getCompleted() { - return completed; + public Long getSuccessful() { + return successful; + } + + public void setSuccessful(Long successful) { + this.successful = successful; + } + + public BigDecimal getTotalAmount() { + return totalAmount; + } + + public void setTotalAmount(BigDecimal totalAmount) { + this.totalAmount = totalAmount; + } + + public BigDecimal getSuccessfulAmount() { + return successfulAmount; + } + + public void setSuccessfulAmount(BigDecimal successfulAmount) { + this.successfulAmount = successfulAmount; + } + + public BigDecimal getPendingAmount() { + return pendingAmount; } - public void setCompleted(Long completed) { - this.completed = completed; + public void setPendingAmount(BigDecimal pendingAmount) { + this.pendingAmount = pendingAmount; } - public BigDecimal getTotal_amount() { - return total_amount; + public BigDecimal getFailedAmount() { + return failedAmount; } - public void setTotal_amount(BigDecimal total_amount) { - this.total_amount = total_amount; + public void setFailedAmount(BigDecimal failedAmount) { + this.failedAmount = failedAmount; } - public BigDecimal getCompleted_amount() { - return completed_amount; + public String getFile() { + return file; } - public void setCompleted_amount(BigDecimal completed_amount) { - this.completed_amount = completed_amount; + public void setFile(String file) { + this.file = file; } - public BigDecimal getOngoing_amount() { - return ongoing_amount; + public String getNotes() { + return notes; } - public void setOngoing_amount(BigDecimal ongoing_amount) { - this.ongoing_amount = ongoing_amount; + public void setNotes(String notes) { + this.notes = notes; } - public BigDecimal getFailed_amount() { - return failed_amount; + public String getCreated_at() { + return created_at; } - public void setFailed_amount(BigDecimal failed_amount) { - this.failed_amount = failed_amount; + public void setCreated_at(String created_at) { + this.created_at = created_at; } - public String getResult_file() { - return result_file; + public String getStatus() { + return status; } - public void setResult_file(String result_file) { - this.result_file = result_file; + public void setStatus(String status) { + this.status = status; } - public Date getResultGeneratedAt() { - return resultGeneratedAt; + public String getModes() { + return modes; } - public void setResultGeneratedAt(Date resultGeneratedAt) { - this.resultGeneratedAt = resultGeneratedAt; + public void setModes(String modes) { + this.modes = modes; } - public String getNote() { - return note; + public String getPurpose() { + return purpose; } - public void setNote(String note) { - this.note = note; + public void setPurpose(String purpose) { + this.purpose = purpose; } } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java index 3e41c86f..bdc14240 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java @@ -27,6 +27,11 @@ public void setup() { List successSubBatches = (List) variables.get(INIT_SUCCESS_SUB_BATCHES); List failureSubBatches = (List) variables.get(INIT_FAILURE_SUB_BATCHES); + for (int i = 0; i < successSubBatches.size(); i++) { + String initFile = successSubBatches.remove(i); + successSubBatches.add(i, String.format("Result_%s", initFile)); + } + List mergeFileList = (List) variables.get(MERGE_FILE_LIST); if (mergeFileList == null) { mergeFileList = new ArrayList<>(); From 038cc387430af4e7e58c05cbe6329dfeef98b720 Mon Sep 17 00:00:00 2001 From: DanishJamal Date: Tue, 20 Sep 2022 19:45:09 +0530 Subject: [PATCH 034/156] Bug fixed in splitting worker --- .../mifos/processor/bulk/zeebe/worker/SplittingWorker.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java index 06a4e328..bf972c9b 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java @@ -8,6 +8,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import static org.mifos.processor.bulk.camel.config.CamelProperties.*; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @@ -43,10 +44,12 @@ public void setup() { } Boolean subBatchCreated = exchange.getProperty(SUB_BATCH_CREATED, Boolean.class); + Optional subBatchCreatedOptional = Optional.ofNullable(subBatchCreated); List serverSubBatchFileList = exchange.getProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, List.class); - if (!subBatchCreated && serverSubBatchFileList.isEmpty()) { + if (subBatchCreatedOptional.isPresent() && !subBatchCreatedOptional.get() && serverSubBatchFileList.isEmpty()) { // if no sub-batches is created, insert the original filename in sub batch array serverSubBatchFileList.add(filename); + subBatchCreated = false; } variables.put(SPLITTING_FAILED, false); From 336f864458ed38f4b0a24a977323398e4c96462f Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Wed, 21 Sep 2022 18:25:38 +0530 Subject: [PATCH 035/156] Fixes #27 Invalid RESULT_FILE zeebe variable value (#29) * Bug fixed in splitting worker * Fixes #27: Bug fixed in setting RESULT_FILE url * Removed redundant optional from splitting worker --- .../processor/bulk/camel/routes/MergeBackRoute.java | 12 ++++++++++-- .../processor/bulk/camel/routes/SplittingRoute.java | 1 + .../processor/bulk/zeebe/worker/SplittingWorker.java | 6 ++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java index 83a16598..581a17a7 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java @@ -1,5 +1,7 @@ package org.mifos.processor.bulk.camel.routes; +import com.amazonaws.services.dynamodbv2.xspec.L; +import org.apache.camel.Exchange; import org.mifos.processor.bulk.utility.Utils; import org.springframework.stereotype.Component; import java.io.File; @@ -33,7 +35,8 @@ public void configure() throws Exception { .process(exchange -> { exchange.setProperty(MERGE_FAILED, false); exchange.setProperty(MERGE_COMPLETED, true); - exchange.setProperty(RESULT_FILE, exchange.getProperty(MERGE_FILE_LIST, List.class).get(0)); + String resultFile = (String) exchange.getProperty(MERGE_FILE_LIST, List.class).get(0); + setResultFileProperty(exchange, resultFile); }) .otherwise() .to("direct:start-merge") @@ -77,7 +80,7 @@ public void configure() throws Exception { if (mergeList.size() == 1) { exchange.setProperty(MERGE_FAILED, false); exchange.setProperty(MERGE_COMPLETED, true); - exchange.setProperty(RESULT_FILE, Utils.getAwsFileUrl(awsS3BaseUrl,mergedFileServerName)); + setResultFileProperty(exchange, mergedFileServerName); } else { exchange.setProperty(MERGE_COMPLETED, false); } @@ -107,4 +110,9 @@ public void configure() throws Exception { .to("direct:download-file") // downloading second file .setProperty(FILE_2, exchangeProperty(LOCAL_FILE_PATH)); } + + // set RESULT_FILE exchange property to the file url + public void setResultFileProperty(Exchange exchange, String fileName) { + exchange.setProperty(RESULT_FILE, Utils.getAwsFileUrl(awsS3BaseUrl, fileName)); + } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java index d8e48043..b0373adb 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java @@ -54,6 +54,7 @@ public void configure() throws Exception { if (lines.size() <= subBatchSize) { exchange.setProperty(SUB_BATCH_CREATED, false); + exchange.setProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, new ArrayList()); logger.info("Skipping creating sub batch, as batch size is less than configured sub-batch size"); return; } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java index bf972c9b..8997cd5c 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java @@ -4,12 +4,10 @@ import org.apache.camel.support.DefaultExchange; import org.mifos.processor.bulk.camel.routes.RouteId; import org.springframework.stereotype.Component; - import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; - import static org.mifos.processor.bulk.camel.config.CamelProperties.*; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @@ -44,9 +42,9 @@ public void setup() { } Boolean subBatchCreated = exchange.getProperty(SUB_BATCH_CREATED, Boolean.class); - Optional subBatchCreatedOptional = Optional.ofNullable(subBatchCreated); List serverSubBatchFileList = exchange.getProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, List.class); - if (subBatchCreatedOptional.isPresent() && !subBatchCreatedOptional.get() && serverSubBatchFileList.isEmpty()) { + if (subBatchCreated != null && !subBatchCreated && + serverSubBatchFileList != null && serverSubBatchFileList.isEmpty()) { // if no sub-batches is created, insert the original filename in sub batch array serverSubBatchFileList.add(filename); subBatchCreated = false; From e1d57e82ff386e0e315607956beba1a34289adaa Mon Sep 17 00:00:00 2001 From: Ankita Ranjan <92036361+ankita10r@users.noreply.github.com> Date: Thu, 22 Sep 2022 16:13:11 +0530 Subject: [PATCH 036/156] Adding logic for callback after aggregate api (#28) --- .../bulk/camel/config/CamelProperties.java | 2 + .../camel/routes/ProcessorStartRoute.java | 7 ++ .../processor/bulk/camel/routes/RouteId.java | 1 + .../bulk/camel/routes/SendCallbackRoute.java | 80 +++++++++++++++++++ .../processor/bulk/zeebe/ZeebeVariables.java | 10 +++ .../bulk/zeebe/worker/SendCallbackWorker.java | 48 +++++++++++ .../processor/bulk/zeebe/worker/Worker.java | 1 + src/main/resources/application.yaml | 3 + 8 files changed, 152 insertions(+) create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index c498a04c..c14184a4 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -35,6 +35,8 @@ private CamelProperties() {} public static final String BATCH_STATUS_FAILED = "batchStatusFailed"; + public static final String CALLBACK_RESPONSE_CODE = "responseCode"; + public static final String BATCH_REQUEST_TYPE = "batchRequestType"; } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 0cdb3304..104d93c2 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -146,6 +146,9 @@ private void setup() { logger.info("File uploaded {}", nm); + //extracting and setting callback Url + String callbackUrl = exchange.getIn().getHeader("Callback-Url", String.class); + exchange.setProperty(CALLBACK_URL,callbackUrl); Map variables = new HashMap<>(); variables.put(BATCH_ID, batchId); @@ -153,6 +156,7 @@ private void setup() { variables.put(REQUEST_ID, requestId); variables.put(PURPOSE, purpose); variables.put(TENANT_ID, exchange.getProperty(TENANT_NAME)); + variables.put(CALLBACK_URL,callbackUrl); setConfigProperties(variables); JSONObject response = new JSONObject(); @@ -204,6 +208,9 @@ private Map setConfigProperties(Map variables) { variables.put(MAX_STATUS_RETRY, maxThresholdCheckRetry); variables.put(SUCCESS_THRESHOLD, successThreshold); variables.put(THRESHOLD_DELAY, Utils.getZeebeTimerValue(thresholdCheckDelay)); + variables.put(BULK_NOTIF_SUCCESS,false); + variables.put(BULK_NOTIF_FAILURE,false); + return variables; } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java b/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java index 8e182df0..74819193 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java @@ -8,6 +8,7 @@ public enum RouteId { SPLITTING("direct:splitting"), FORMATTING("direct:formatting"), BATCH_STATUS("direct:batchStatus"), + SEND_CALLBACK("direct:sendCallback"), MERGE_BACK("direct:mergeSubBatch"), INIT_SUB_BATCH("direct:init-sub-batches"); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java new file mode 100644 index 00000000..c2c25af6 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java @@ -0,0 +1,80 @@ +package org.mifos.processor.bulk.camel.routes; + +import org.apache.camel.Exchange; +import org.apache.camel.LoggingLevel; +import org.apache.camel.model.dataformat.JsonLibrary; +import org.apache.camel.spi.AsEndpointUri; +import org.mifos.processor.bulk.schema.BatchDTO; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; + +@Component +public class SendCallbackRoute extends BaseRouteBuilder { + + + @Value("${callback.url}") + private String callbackUrl; + + @Override + public void configure() throws Exception { + + from("rest:get:test/send/callback") + .to(RouteId.SEND_CALLBACK.getValue()); + + /** + * Base route for kicking off callback. Performs below tasks. + * Sends Callback to the set url + * Checks of response code is anything not 2xx then retries + */ + + from(RouteId.SEND_CALLBACK.getValue()) + .id(RouteId.SEND_CALLBACK.getValue()) + .log("Starting route " + RouteId.SEND_CALLBACK.name()) + .log("Sending callback for Batch Processing") + .setBody(exchange -> { + String body = "Batch Aggregation API was complete with %: " + + exchange.getProperty(SUCCESS_RATE); + callbackUrl = exchange.getProperty(CALLBACK_URL).toString(); + return body; + }) + .toD("${header.callbackUrl}?bridgeEndpoint=true&throwExceptionOnFailure=false") + .choice() + .when(header("CamelHttpResponseCode").startsWith("2")) + .log(LoggingLevel.INFO, "Callback sending was successful") + .process(exchange -> { + exchange.setProperty(CALLBACK_RESPONSE_CODE, exchange.getIn() + .getHeader(Exchange.HTTP_RESPONSE_CODE)); + exchange.setProperty(CALLBACK_RETRY, 1); + exchange.setProperty(CALLBACK_SUCCESS, true); + + }) + .otherwise() + .log(LoggingLevel.ERROR, "Callback request was unsuccessful") + .process(exchange -> { + if (exchange.getProperty(CALLBACK_RETRY).equals(MAX_STATUS_RETRY)) { + logger.info("Retry Exhausted, setting Callback as Failed"); + } else { + int retry = (int) exchange.getProperty(CALLBACK_RETRY); + retry++; + logger.info("Retry Left {}, Setting Callback as Failed and Retrying...", + ((int) exchange.getProperty(MAX_STATUS_RETRY) - retry)); + exchange.setProperty(CALLBACK_RETRY, retry); + exchange.setProperty(CALLBACK_RESPONSE_CODE, exchange.getIn() + .getHeader(Exchange.HTTP_RESPONSE_CODE)); + + } + exchange.setProperty(CALLBACK_SUCCESS, false); + exchange.setProperty(ERROR_DESCRIPTION, exchange.getIn().getBody(String.class)); + exchange.setProperty(ERROR_CODE, exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE)); + }); + + //for temporary callback simulation + from("rest:post:/simulate") + .log("Reached Simulation"); + } + + +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 65156399..07312f58 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -87,6 +87,8 @@ private ZeebeVariables() { public static final String RETRY = "retry"; + public static final String CALLBACK_RETRY = "callbackRetryCount"; + public static final String SUCCESS_THRESHOLD = "successThreshold"; public static final String SUCCESS_RATE = "successRate"; @@ -98,4 +100,12 @@ private ZeebeVariables() { public static final String THRESHOLD_DELAY = "thresholdDelay"; public static final String PAYMENT_MODE = "paymentMode"; + + public static final String CALLBACK_SUCCESS = "callbackSuccessful"; + + public static final String CALLBACK_URL = "callbackUrl"; + + public static final String BULK_NOTIF_SUCCESS = "isNotificationsSuccessEnabled"; + + public static final String BULK_NOTIF_FAILURE = "isNotificationsFailureEnabled"; } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java new file mode 100644 index 00000000..58b8f578 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java @@ -0,0 +1,48 @@ +package org.mifos.processor.bulk.zeebe.worker; + +import org.apache.camel.Exchange; +import org.apache.camel.support.DefaultExchange; +import org.mifos.processor.bulk.camel.routes.RouteId; +import org.springframework.stereotype.Component; + +import java.util.Map; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_STATUS_FAILED; +import static org.mifos.processor.bulk.camel.config.CamelProperties.CALLBACK_RESPONSE_CODE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; + +@Component +public class SendCallbackWorker extends BaseWorker { + + @Override + public void setup() { + newWorker(Worker.SEND_CALLBACK, (client, job) -> { + Map variables = job.getVariablesAsMap(); + + int retry = (int) variables.getOrDefault(CALLBACK_RETRY, 0); + + Exchange exchange = new DefaultExchange(camelContext); + exchange.setProperty(MAX_STATUS_RETRY,variables.get(MAX_STATUS_RETRY)); + exchange.setProperty(CALLBACK_RETRY,retry); + exchange.setProperty(CALLBACK_URL,variables.get(CALLBACK_URL)); + exchange.setProperty(SUCCESS_RATE,variables.get(SUCCESS_RATE)); + sendToCamelRoute(RouteId.SEND_CALLBACK, exchange); + + Boolean callbackSuccess = exchange.getProperty(CALLBACK_SUCCESS, Boolean.class); + if (callbackSuccess == null || !callbackSuccess) { + variables.put(ERROR_CODE, exchange.getProperty(ERROR_CODE)); + variables.put(ERROR_DESCRIPTION, exchange.getProperty(ERROR_DESCRIPTION)); + logger.info("Error: {}, {}", variables.get(ERROR_CODE), variables.get(ERROR_DESCRIPTION)); + } else { + variables.put(CALLBACK_SUCCESS,true); + } + + variables.put(CALLBACK_RETRY,exchange.getProperty(CALLBACK_RETRY)); + variables.put(CALLBACK_RESPONSE_CODE,exchange.getProperty(CALLBACK_RESPONSE_CODE)); + + logger.info("Retry: {} and Response Code {}", exchange.getProperty(CALLBACK_RETRY), exchange.getProperty(CALLBACK_RESPONSE_CODE)); + client.newCompleteCommand(job.getKey()).variables(variables).send(); + }); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java index c99a6221..0e4c79c2 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java @@ -8,6 +8,7 @@ public enum Worker { SPLITTING("splitting"), FORMATTING("formatting"), BATCH_STATUS("batchStatus"), + SEND_CALLBACK("sendCallback"), MERGE_BACK("mergeSubBatch"), INIT_SUB_BATCH("initSubBatch"); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 41b2ebbe..25aadd48 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -89,3 +89,6 @@ config: success-threshold: 95 # in percentage max-retry: 4 delay: 60 # in seconds + +callback: + url: "http://httpstat.us/503" From e676a62f9ac7a17378c3057eec70afd09075e5f6 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Fri, 23 Sep 2022 13:12:57 +0530 Subject: [PATCH 037/156] Mulitpart file unmarshling for new specs (#33) --- .../camel/routes/ProcessorStartRoute.java | 2 + .../processor/bulk/schema/TestSchema.java | 149 ------------------ .../processor/bulk/schema/Transaction.java | 12 +- 3 files changed, 11 insertions(+), 152 deletions(-) delete mode 100644 src/main/java/org/mifos/processor/bulk/schema/TestSchema.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 104d93c2..c39ab39b 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -64,6 +64,7 @@ private void setup() { from("rest:POST:/batchtransactions") .id("rest:POST:/batchtransactions") .log("Starting route rest:POST:/batchtransactions") + .unmarshal().mimeMultipart("multipart/*") .to("direct:validate-tenant") .process(exchange -> { String filename = exchange.getIn().getHeader("filename", String.class); @@ -105,6 +106,7 @@ private void setup() { .log("Validating tenant") .process(exchange -> { String tenantName = exchange.getIn().getHeader("Platform-TenantId", String.class); + // validation is disabled for now /*if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { throw new Exception("Invalid tenant value."); }*/ diff --git a/src/main/java/org/mifos/processor/bulk/schema/TestSchema.java b/src/main/java/org/mifos/processor/bulk/schema/TestSchema.java deleted file mode 100644 index 8012724a..00000000 --- a/src/main/java/org/mifos/processor/bulk/schema/TestSchema.java +++ /dev/null @@ -1,149 +0,0 @@ -package org.mifos.processor.bulk.schema; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - -// id,request_id,payment_mode,payer_identifier_type, -// payer_identifier,payee_identifier_type,payee_identifier, -// amount,currency,note - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonPropertyOrder({ "id", "request_id", "payment_mode", "payer_identifier_type", "payer_identifier", "payee_identifier_type", "payee_identifier", "currency","amount", "note", "program_shortcode", }) -public class TestSchema { - - @JsonProperty("id") - private int id; - - @JsonProperty("request_id") - private String requestId; - - @JsonProperty("payment_mode") - private String paymentMode; - - @JsonProperty("payer_identifier_type") - private String payerIdentifierType; - - @JsonProperty("payer_identifier") - private String payerIdentifier; - - @JsonProperty("payee_identifier_type") - private String payeeIdentifierType; - - @JsonProperty("payee_identifier") - private String payeeIdentifier; - - @JsonProperty("currency") - private String currency; - - @JsonProperty("amount") - private String amount; - - @JsonProperty("note") - private String note; - - @JsonProperty("program_shortcode") - private String programShortCode; - - @JsonProperty("cycle") - private String cycle; - - public TestSchema() { - } - - public int getId() { - return id; - } - - public void setId(int id) { - this.id = id; - } - - public String getRequestId() { - return requestId; - } - - public void setRequestId(String requestId) { - this.requestId = requestId; - } - - public String getPaymentMode() { - return paymentMode; - } - - public void setPaymentMode(String paymentMode) { - this.paymentMode = paymentMode; - } - - public String getPayerIdentifierType() { - return payerIdentifierType; - } - - public void setPayerIdentifierType(String payerIdentifierType) { - this.payerIdentifierType = payerIdentifierType; - } - - public String getPayerIdentifier() { - return payerIdentifier; - } - - public void setPayerIdentifier(String payerIdentifier) { - this.payerIdentifier = payerIdentifier; - } - - public String getPayeeIdentifierType() { - return payeeIdentifierType; - } - - public void setPayeeIdentifierType(String payeeIdentifierType) { - this.payeeIdentifierType = payeeIdentifierType; - } - - public String getPayeeIdentifier() { - return payeeIdentifier; - } - - public void setPayeeIdentifier(String payeeIdentifier) { - this.payeeIdentifier = payeeIdentifier; - } - - public String getAmount() { - return amount; - } - - public void setAmount(String amount) { - this.amount = amount; - } - - public String getCurrency() { - return currency; - } - - public void setCurrency(String currency) { - this.currency = currency; - } - - public String getNote() { - return note; - } - - public void setNote(String note) { - this.note = note; - } - - public String getProgramShortCode() { - return programShortCode; - } - - public void setProgramShortCode(String programShortCode) { - this.programShortCode = programShortCode; - } - - public String getCycle() { - return cycle; - } - - public void setCycle(String cycle) { - this.cycle = cycle; - } -} diff --git a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java index 27a79f17..8e2e041e 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java +++ b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java @@ -118,12 +118,18 @@ public void setBatchId(String batchId) { public String toString() { return "Transaction{" + "id=" + id + - ", request_id='" + requestId + '\'' + - ", payment_mode='" + paymentMode + '\'' + - ", account_number='" + accountNumber + '\'' + + ", requestId='" + requestId + '\'' + + ", paymentMode='" + paymentMode + '\'' + + ", accountNumber='" + accountNumber + '\'' + ", amount='" + amount + '\'' + ", currency='" + currency + '\'' + ", note='" + note + '\'' + + ", payerIdentifierType='" + payerIdentifierType + '\'' + + ", payerIdentifier='" + payerIdentifier + '\'' + + ", payeeIdentifierType='" + payeeIdentifierType + '\'' + + ", payeeIdentifier='" + payeeIdentifier + '\'' + + ", programShortCode='" + programShortCode + '\'' + + ", cycle='" + cycle + '\'' + ", batchId='" + batchId + '\'' + '}'; } From 2697859a22f8fc1ed015ee3d66a2552fc217fde4 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Fri, 23 Sep 2022 13:26:04 +0530 Subject: [PATCH 038/156] Null pointer check added in init-sub-batch worker (#34) --- .../mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java index 4029c85a..ed265011 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java @@ -56,8 +56,8 @@ public void setup() { sendToCamelRoute(RouteId.INIT_SUB_BATCH, exchange); - boolean subBatchFailed = exchange.getProperty(INIT_SUB_BATCH_FAILED, Boolean.class); - if (subBatchFailed) { + Boolean subBatchFailed = exchange.getProperty(INIT_SUB_BATCH_FAILED, Boolean.class); + if (subBatchFailed != null && subBatchFailed) { failureSubBatches.add(fileName); } else { successSubBatches.add(fileName); From d8c74ece860d26e0ec6377a1a75c15dec648a069 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Mon, 26 Sep 2022 16:48:19 +0530 Subject: [PATCH 039/156] Updating file status in case of missing payment mode (#36) --- build.gradle | 2 + .../bulk/camel/config/CamelProperties.java | 4 + .../camel/routes/FileProcessingRoute.java | 38 +++++ .../bulk/camel/routes/InitSubBatchRoute.java | 107 ++++++++---- .../processor/bulk/schema/Transaction.java | 157 +++--------------- .../bulk/schema/TransactionResult.java | 42 +++++ .../mifos/processor/bulk/utility/Utils.java | 24 +++ .../bulk/zeebe/worker/MergeBackWorker.java | 4 + 8 files changed, 215 insertions(+), 163 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/schema/TransactionResult.java diff --git a/build.gradle b/build.gradle index d0905e90..1de06a8e 100644 --- a/build.gradle +++ b/build.gradle @@ -42,6 +42,8 @@ dependencies { testImplementation 'org.springframework.boot:spring-boot-starter-test:2.5.2' implementation 'org.apache.camel:camel-jackson:3.4.0' implementation 'org.apache.camel.springboot:camel-mail-starter:3.4.0' + compileOnly 'org.projectlombok:lombok:1.18.24' + annotationProcessor 'org.projectlombok:lombok:1.18.24' } group = 'org.mifos' diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index c14184a4..0bf9f462 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -39,4 +39,8 @@ private CamelProperties() {} public static final String BATCH_REQUEST_TYPE = "batchRequestType"; + public static final String RESULT_TRANSACTION_LIST = "resultTransactionList"; + + public static final String ZEEBE_VARIABLE = "zeebeVariable"; + } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java index 6be82fea..3f17b1f0 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.dataformat.csv.CsvSchema; import org.mifos.processor.bulk.schema.Transaction; +import org.mifos.processor.bulk.schema.TransactionResult; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.io.*; @@ -52,6 +53,27 @@ public void configure() { exchange.setProperty(COMPLETED_AMOUNT, completedAmount); }); + /** + * updates the data in local file + * exchangeInput: + * [LOCAL_FILE_PATH] the absolute path to the csv file + * [RESULT_TRANSACTION_LIST] containing the list of [Transaction] + * [OVERRIDE_HEADER] if set to true will override the header or else use the existing once in csv file + */ + from("direct:update-result-file") + .id("direct:update-result-file") + .log("Starting route direct:update-result-file") + .process(exchange -> { + String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); + List transactionList = exchange.getProperty(RESULT_TRANSACTION_LIST, List.class); + + // getting header + Boolean overrideHeader = exchange.getProperty(OVERRIDE_HEADER, Boolean.class); + + csvWriter(transactionList, TransactionResult.class, csvMapper, overrideHeader, filepath); + }) + .log("Update complete"); + /** * updates the data in local file * exchangeInput: @@ -95,4 +117,20 @@ public void configure() { } }); } + + private void csvWriter(List data, Class tClass, CsvMapper csvMapper, + boolean overrideHeader, String filepath) throws IOException { + CsvSchema csvSchema = csvMapper.schemaFor(tClass); + if (overrideHeader) { + csvSchema = csvSchema.withHeader(); + } else { + csvSchema = csvSchema.withoutHeader(); + } + + File file = new File(filepath); + SequenceWriter writer = csvMapper.writerWithSchemaFor(tClass).with(csvSchema).writeValues(file); + for (T object: data) { + writer.write(object); + } + } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index d41afec4..a2ef5cbd 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -1,5 +1,6 @@ package org.mifos.processor.bulk.camel.routes; +import org.mifos.processor.bulk.schema.TransactionResult; import org.mifos.processor.bulk.utility.Utils; import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.zeebe.BpmnConfig; @@ -7,10 +8,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.util.*; import static org.mifos.processor.bulk.camel.config.CamelProperties.*; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @@ -38,36 +36,89 @@ public void configure() throws Exception { .log("Starting route " + RouteId.INIT_SUB_BATCH.name()) .to("direct:download-file") .to("direct:get-transaction-array") - .to("direct:start-workflow"); + .to("direct:start-workflow-step1"); + + // crates the zeebe variables map and starts the workflow by calling >> direct:start-workflow-step2 + from("direct:start-workflow-step1") + .id("direct:start-flow-step1") + .log("Starting route direct:start-flow-step1") + .process(exchange -> { + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + + Map variables = new HashMap<>(); + variables.put(BATCH_ID, exchange.getProperty(BATCH_ID)); + variables.put(SUB_BATCH_ID, UUID.randomUUID().toString()); + variables.put(FILE_NAME, exchange.getProperty(SERVER_FILE_NAME)); + variables.put(REQUEST_ID, exchange.getProperty(REQUEST_ID)); + variables.put(PURPOSE, exchange.getProperty(PURPOSE)); + variables.put(TOTAL_AMOUNT, exchange.getProperty(TOTAL_AMOUNT)); + variables.put(ONGOING_AMOUNT, exchange.getProperty(ONGOING_AMOUNT)); + variables.put(FAILED_AMOUNT, exchange.getProperty(FAILED_AMOUNT)); + variables.put(COMPLETED_AMOUNT, exchange.getProperty(COMPLETED_AMOUNT)); + variables.put(RESULT_FILE, String.format("Result_%s", + exchange.getProperty(SERVER_FILE_NAME))); + + exchange.setProperty(ZEEBE_VARIABLE, variables); + exchange.setProperty(PAYMENT_MODE, transactionList.get(0).getPaymentMode()); + + }) + .to("direct:start-workflow-step2"); // Loops through each transaction and start the respective workflow - from("direct:start-workflow") - .id("direct:start-flow") - .log("Starting route direct:start-flow") + from("direct:start-workflow-step2") + .id("direct:start-flow-step2") + .log("Starting route direct:start-flow-step2") + .choice() + .when(exchangeProperty(PAYMENT_MODE).isEqualToIgnoreCase("slcb")) .process(exchange -> { String tenantName = exchange.getProperty(TENANT_NAME, String.class); + Map variables = exchange.getProperty(ZEEBE_VARIABLE, Map.class); + variables.put(PAYMENT_MODE, "slcb"); + zeebeProcessStarter.startZeebeWorkflow( + Utils.getTenantSpecificWorkflowId(bpmnConfig.slcbBpmn, tenantName), variables); + exchange.setProperty(INIT_SUB_BATCH_FAILED, false); + }) + .otherwise() + .to("direct:payment-mode-missing") + .setProperty(INIT_SUB_BATCH_FAILED, constant(true)) + .endChoice(); + + from("direct:payment-mode-missing") + .id("direct:payment-mode-missing") + .log("Starting route direct:payment-mode-missing") + .process(exchange -> { + String serverFileName = exchange.getProperty(SERVER_FILE_NAME, String.class); + String resultFile = String.format("Result_%s", serverFileName); + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); - if (transactionList.get(0).getPaymentMode().equalsIgnoreCase("slcb")) { - Map variables = new HashMap<>(); - variables.put(BATCH_ID, exchange.getProperty(BATCH_ID)); - variables.put(SUB_BATCH_ID, UUID.randomUUID().toString()); - variables.put(FILE_NAME, exchange.getProperty(SERVER_FILE_NAME)); - variables.put(REQUEST_ID, exchange.getProperty(REQUEST_ID)); - variables.put(PURPOSE, exchange.getProperty(PURPOSE)); - variables.put(TOTAL_AMOUNT, exchange.getProperty(TOTAL_AMOUNT)); - variables.put(ONGOING_AMOUNT, exchange.getProperty(ONGOING_AMOUNT)); - variables.put(FAILED_AMOUNT, exchange.getProperty(FAILED_AMOUNT)); - variables.put(COMPLETED_AMOUNT, exchange.getProperty(COMPLETED_AMOUNT)); - variables.put(RESULT_FILE, String.format("Result_%s", - exchange.getProperty(SERVER_FILE_NAME))); - variables.put(PAYMENT_MODE, "slcb"); - - zeebeProcessStarter.startZeebeWorkflow( - Utils.getTenantSpecificWorkflowId(bpmnConfig.slcbBpmn, tenantName), variables); - } + List transactionResultList = updateTransactionStatusToFailed(transactionList); + exchange.setProperty(RESULT_TRANSACTION_LIST, transactionResultList); + exchange.setProperty(RESULT_FILE, resultFile); + }) + // setting localfilpath as result file to make sure result file is uploaded + .setProperty(LOCAL_FILE_PATH, exchangeProperty(RESULT_FILE)) + .setProperty(OVERRIDE_HEADER, constant(true)) + .process(exchange -> { + logger.info("A1 {}", exchange.getProperty(RESULT_FILE)); + logger.info("A2 {}", exchange.getProperty(LOCAL_FILE_PATH)); + logger.info("A3 {}", exchange.getProperty(OVERRIDE_HEADER)); + }) + .to("direct:update-result-file") + .to("direct:upload-file"); + } - exchange.setProperty(INIT_SUB_BATCH_FAILED, false); - }); + // update Transactions status to failed + private List updateTransactionStatusToFailed(List transactionList) { + List transactionResultList = new ArrayList<>(); + for (Transaction transaction : transactionList) { + TransactionResult transactionResult = Utils.mapToResultDTO(transaction); + transactionResult.setErrorCode("404"); + transactionResult.setErrorDescription("Payment mode not configured"); + transactionResult.setStatus("Failed"); + transactionResultList.add(transactionResult); + } + + return transactionResultList; } } diff --git a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java index 8e2e041e..e1c37d0f 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java +++ b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java @@ -1,8 +1,14 @@ package org.mifos.processor.bulk.schema; -import com.fasterxml.jackson.annotation.*; -import jdk.nashorn.internal.ir.annotations.Ignore; - +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter @JsonIgnoreProperties(ignoreUnknown = true) @JsonPropertyOrder({ "id", "request_id", "payment_mode", "account_number", "payer_identifier_type", "payer_identifier", "payee_identifier_type", "payee_identifier", "amount", "currency", "note", "program_shortcode", "cycle" }) public class Transaction implements CsvSchema { @@ -24,6 +30,7 @@ public class Transaction implements CsvSchema { @JsonProperty("currency") private String currency; + @JsonProperty("note") private String note; @@ -48,138 +55,19 @@ public class Transaction implements CsvSchema { @JsonIgnore private String batchId; - public int getId() { - return id; - } - - public void setId(int id) { - this.id = id; - } - - public String getRequestId() { - return requestId; - } - - public void setRequestId(String requestId) { - this.requestId = requestId; - } - - public String getPaymentMode() { - return paymentMode; - } - - public void setPaymentMode(String paymentMode) { - this.paymentMode = paymentMode; - } - - public String getAccountNumber() { - return accountNumber; - } - - public void setAccountNumber(String accountNumber) { - this.accountNumber = accountNumber; - } - - public String getAmount() { - return amount; - } - - public void setAmount(String amount) { - this.amount = amount; - } - - public String getCurrency() { - return currency; - } - - public void setCurrency(String currency) { - this.currency = currency; - } - - public String getNote() { - return note; - } - - public void setNote(String note) { - this.note = note; - } - - @JsonIgnore - public String getBatchId() { - return batchId; - } - - @JsonIgnore - public void setBatchId(String batchId) { - this.batchId = batchId; - } - @Override public String toString() { - return "Transaction{" + - "id=" + id + - ", requestId='" + requestId + '\'' + - ", paymentMode='" + paymentMode + '\'' + - ", accountNumber='" + accountNumber + '\'' + - ", amount='" + amount + '\'' + - ", currency='" + currency + '\'' + - ", note='" + note + '\'' + - ", payerIdentifierType='" + payerIdentifierType + '\'' + - ", payerIdentifier='" + payerIdentifier + '\'' + - ", payeeIdentifierType='" + payeeIdentifierType + '\'' + - ", payeeIdentifier='" + payeeIdentifier + '\'' + - ", programShortCode='" + programShortCode + '\'' + - ", cycle='" + cycle + '\'' + - ", batchId='" + batchId + '\'' + - '}'; - } - - public String getPayerIdentifierType() { - return payerIdentifierType; - } - - public void setPayerIdentifierType(String payerIdentifierType) { - this.payerIdentifierType = payerIdentifierType; - } - - public String getPayerIdentifier() { - return payerIdentifier; - } - - public void setPayerIdentifier(String payerIdentifier) { - this.payerIdentifier = payerIdentifier; - } - - public String getPayeeIdentifierType() { - return payeeIdentifierType; - } - - public void setPayeeIdentifierType(String payeeIdentifierType) { - this.payeeIdentifierType = payeeIdentifierType; - } - - public String getPayeeIdentifier() { - return payeeIdentifier; - } - - public void setPayeeIdentifier(String payeeIdentifier) { - this.payeeIdentifier = payeeIdentifier; - } - - public String getProgramShortCode() { - return programShortCode; - } - - public void setProgramShortCode(String programShortCode) { - this.programShortCode = programShortCode; - } - - public String getCycle() { - return cycle; - } - - public void setCycle(String cycle) { - this.cycle = cycle; + StringBuilder buffer = new StringBuilder("Transaction{"); + buffer.append("id=").append(id); + buffer.append(", request_id='").append(requestId); + buffer.append(", payment_mode='").append(paymentMode); + buffer.append(", account_number='").append(accountNumber); + buffer.append(", amount='").append(amount); + buffer.append(", currency='").append(currency); + buffer.append(", note='").append(note); + buffer.append(", batchId='").append(batchId); + buffer.append(", status='").append(id).append('}'); + return buffer.toString(); } @JsonIgnore @@ -191,7 +79,6 @@ public String getCsvString() { @JsonIgnore @Override public String getCsvHeader() { - return "id,request_id,payment_mode,account_number,amount,currency,note"; + return "id,request_id,payment_mode,account_number,amount,currency,note,status"; } - } diff --git a/src/main/java/org/mifos/processor/bulk/schema/TransactionResult.java b/src/main/java/org/mifos/processor/bulk/schema/TransactionResult.java new file mode 100644 index 00000000..f3c6911b --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/TransactionResult.java @@ -0,0 +1,42 @@ +package org.mifos.processor.bulk.schema; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonPropertyOrder({ "id", "request_id", "payment_mode", "payer_identifier_type", "payer_identifier", + "payee_identifier_type", "payee_identifier", "amount", "currency", "note", "program_shortcode", "cycle", + "status", "error_code", "error_description"}) +public class TransactionResult extends Transaction { + + @JsonProperty("status") + private String status; + + @JsonProperty("error_code") + private String errorCode; + + @JsonProperty("error_description") + private String errorDescription; + + @JsonProperty("account_number") + @JsonIgnore + private String accountNumber; + + @JsonIgnore + @Override + public void setAccountNumber(String accountNumber) { + super.setAccountNumber(accountNumber); + } + + @JsonIgnore + @Override + public String getAccountNumber() { + return super.getAccountNumber(); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/utility/Utils.java b/src/main/java/org/mifos/processor/bulk/utility/Utils.java index 126c72d2..52cd353c 100644 --- a/src/main/java/org/mifos/processor/bulk/utility/Utils.java +++ b/src/main/java/org/mifos/processor/bulk/utility/Utils.java @@ -1,5 +1,8 @@ package org.mifos.processor.bulk.utility; +import org.mifos.processor.bulk.schema.Transaction; +import org.mifos.processor.bulk.schema.TransactionResult; + import javax.net.ssl.*; import java.io.*; import java.net.URL; @@ -66,4 +69,25 @@ public static String getZeebeTimerValue(int timer) { return String.format("PT%sS", timer); } + public static TransactionResult mapToResultDTO(Transaction transaction) { + TransactionResult transactionResult = new TransactionResult(); + transactionResult.setId(transaction.getId()); + transactionResult.setRequestId(transaction.getRequestId()); + transactionResult.setPaymentMode(transaction.getPaymentMode()); + transactionResult.setPayerIdentifierType("accountNumber"); + transactionResult.setPayerIdentifier(transaction.getPayerIdentifier()); + transactionResult.setAmount(transaction.getAmount()); + transactionResult.setCurrency(transactionResult.getCurrency()); + transactionResult.setNote(transactionResult.getNote()); + transactionResult.setPayeeIdentifierType("accountNumber"); + if (transaction.getAccountNumber() != null) { + transactionResult.setPayeeIdentifier(transaction.getAccountNumber()); + } else { + transactionResult.setPayeeIdentifier(transaction.getPayeeIdentifier()); + } + transactionResult.setProgramShortCode(transaction.getProgramShortCode()); + transactionResult.setCycle(transactionResult.getCycle()); + return transactionResult; + } + } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java index bdc14240..3271fbaa 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java @@ -31,6 +31,10 @@ public void setup() { String initFile = successSubBatches.remove(i); successSubBatches.add(i, String.format("Result_%s", initFile)); } + for (int i = 0; i < failureSubBatches.size(); i++) { + String initFile = failureSubBatches.remove(i); + failureSubBatches.add(i, String.format("Result_%s", initFile)); + } List mergeFileList = (List) variables.get(MERGE_FILE_LIST); if (mergeFileList == null) { From 5961951019148a9c3d1ec514dd4cf9506374f5ec Mon Sep 17 00:00:00 2001 From: Manoj <56669674+fynmanoj@users.noreply.github.com> Date: Fri, 7 Oct 2022 15:44:41 +0530 Subject: [PATCH 040/156] created-pr-guidelines-template --- .github/pull_request_template.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..98c407da --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,20 @@ +## Description + +* Describe the changes made and why they were made. +* Add a link to teh design document or include the design bullet points related to this PR here. + + _(Ignore if these details are present on the associated JIRA ticket)_ + +## Checklist + +Please make sure these boxes are checked before submitting your pull request - thanks! + +- [ ] Design related bullet points or design document link related to this PR added in the description above. + +- [ ] Updated corresponding Postman Collection or Api documentation for the changes in this PR. + +- [ ] Create/update unit or integration tests for verifying the changes made. + +- [ ] Add required Swagger annotation and update API documentation with details of any API changes if applicable + +- [ ] Followed the naming conventions as given in https://docs.google.com/document/d/1Q4vaMSzrTxxh9TS0RILuNkSkYCxotuYk1Xe0CMIkkCU/edit?usp=sharing From 8065447df771e116504ccbede54df1fd0c26d062 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Fri, 7 Oct 2022 15:54:07 +0530 Subject: [PATCH 041/156] Older endpoint reborn (#39) --- .../org/mifos/processor/bulk/HealthCheck.java | 28 +++-- .../mifos/processor/bulk/kafka/Consumers.java | 118 ++++++++++++++++++ .../kafka/config/KafkaConsumerConfig.java | 40 ++++++ .../kafka/config/KafkaProducerConfig.java | 35 ++++++ .../bulk/kafka/config/KafkaTopicConfig.java | 40 ++++++ .../bulk/schema/TransactionOlder.java | 14 +++ src/main/resources/application.yaml | 7 ++ 7 files changed, 272 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/kafka/Consumers.java create mode 100644 src/main/java/org/mifos/processor/bulk/kafka/config/KafkaConsumerConfig.java create mode 100644 src/main/java/org/mifos/processor/bulk/kafka/config/KafkaProducerConfig.java create mode 100644 src/main/java/org/mifos/processor/bulk/kafka/config/KafkaTopicConfig.java create mode 100644 src/main/java/org/mifos/processor/bulk/schema/TransactionOlder.java diff --git a/src/main/java/org/mifos/processor/bulk/HealthCheck.java b/src/main/java/org/mifos/processor/bulk/HealthCheck.java index f2ee6569..ef0edd9e 100644 --- a/src/main/java/org/mifos/processor/bulk/HealthCheck.java +++ b/src/main/java/org/mifos/processor/bulk/HealthCheck.java @@ -8,7 +8,7 @@ import org.apache.camel.LoggingLevel; import org.apache.camel.builder.RouteBuilder; import org.mifos.processor.bulk.file.FileTransferService; -import org.mifos.processor.bulk.schema.Transaction; +import org.mifos.processor.bulk.schema.TransactionOlder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -19,8 +19,9 @@ import java.util.UUID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; + @Component -@Deprecated public class HealthCheck extends RouteBuilder { private Logger logger = LoggerFactory.getLogger(this.getClass()); @@ -44,6 +45,12 @@ public class HealthCheck extends RouteBuilder { @Value("${zeebe.client.evenly-allocated-max-jobs}") private int workerMaxJobs; + @Value(value = "${kafka.topic.gsma.name}") + private String gsmaTopicName; + + @Value(value = "${kafka.topic.slcb.name}") + private String slcbTopicName; + @Override public void configure() { from("rest:GET:/") @@ -56,24 +63,25 @@ public void configure() { .process(exchange -> { String fileName = exchange.getIn().getHeader("fileName", String.class); String batchId = UUID.randomUUID().toString(); + exchange.setProperty(BATCH_ID, batchId); // TODO: How to get sender information? Hard coded in Channel connector? byte[] csvFile = fileTransferService.downloadFile(fileName, bucketName); CsvSchema schema = CsvSchema.emptySchema().withHeader(); - MappingIterator readValues = csvMapper.readerWithSchemaFor(Transaction.class).with(schema).readValues(csvFile); + MappingIterator readValues = csvMapper.readerWithSchemaFor(TransactionOlder.class).with(schema).readValues(csvFile); - /*while (readValues.hasNext()) { - Transaction current = readValues.next(); + while (readValues.hasNext()) { + TransactionOlder current = readValues.next(); current.setBatchId(batchId); - System.out.println(objectMapper.writeValueAsString(current)); - if (current.getPayment_mode().equals("gsma")) + logger.info("Writing string in kafka {}", objectMapper.writeValueAsString(current)); + if (current.getPaymentMode().equals("gsma") || current.getPaymentMode().equals("afrimoney")) kafkaTemplate.send(gsmaTopicName, objectMapper.writeValueAsString(current)); - else if (current.getPayment_mode().equals("sclb")) + else if (current.getPaymentMode().equals("sclb")) kafkaTemplate.send(slcbTopicName, objectMapper.writeValueAsString(current)); - }*/ + } }) .setHeader(Exchange.HTTP_RESPONSE_CODE, constant(200)) - .setBody(constant("")); + .setBody(exchange -> exchange.getProperty(BATCH_ID)); } } diff --git a/src/main/java/org/mifos/processor/bulk/kafka/Consumers.java b/src/main/java/org/mifos/processor/bulk/kafka/Consumers.java new file mode 100644 index 00000000..ce3a78db --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/kafka/Consumers.java @@ -0,0 +1,118 @@ +package org.mifos.processor.bulk.kafka; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.mifos.connector.common.channel.dto.TransactionChannelRequestDTO; +import org.mifos.connector.common.gsma.dto.GSMATransaction; +import org.mifos.connector.common.gsma.dto.GsmaParty; +import org.mifos.connector.common.mojaloop.dto.MoneyData; +import org.mifos.connector.common.mojaloop.dto.Party; +import org.mifos.connector.common.mojaloop.dto.PartyIdInfo; +import org.mifos.connector.common.mojaloop.dto.TransactionType; +import org.mifos.connector.common.mojaloop.type.IdentifierType; +import org.mifos.processor.bulk.schema.TransactionOlder; +import org.mifos.processor.bulk.zeebe.ZeebeProcessStarter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.mifos.connector.common.mojaloop.type.InitiatorType.CONSUMER; +import static org.mifos.connector.common.mojaloop.type.Scenario.TRANSFER; +import static org.mifos.connector.common.mojaloop.type.TransactionRole.PAYER; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.GSMA_CHANNEL_REQUEST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INITIATOR_FSPID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.IS_RTP_REQUEST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_ID_TYPE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_FSPID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TRANSACTION_TYPE; + +@Service +public class Consumers { + + + @Value("${bpmn.flows.international-remittance-payer}") + private String internationalRemittancePayer; + + @Autowired + private ObjectMapper objectMapper; + + @Autowired + private ZeebeProcessStarter zeebeProcessStarter; + + @KafkaListener(topics = "${kafka.topic.gsma.name}", groupId = "group_id") + public void listenTopicGsma(String message) throws JsonProcessingException { + System.out.println("Received Message in topic GSMA and group group_id: " + message); + + TransactionOlder transaction = objectMapper.readValue((String) message, TransactionOlder.class); + String tenantId = "ibank-usa"; + + GSMATransaction gsmaChannelRequest = new GSMATransaction(); + gsmaChannelRequest.setAmount(transaction.getAmount()); + gsmaChannelRequest.setCurrency(transaction.getCurrency()); + gsmaChannelRequest.setRequestingLei("ibank-usa"); + gsmaChannelRequest.setReceivingLei("ibank-india"); + GsmaParty creditParty = new GsmaParty(); + creditParty.setKey("msisdn"); + creditParty.setValue(transaction.getAccountNumber()); + GsmaParty debitParty = new GsmaParty(); + debitParty.setKey("msisdn"); + debitParty.setValue(transaction.getAccountNumber()); + gsmaChannelRequest.setCreditParty(new GsmaParty[]{creditParty}); + gsmaChannelRequest.setDebitParty(new GsmaParty[]{debitParty}); +// gsmaChannelRequest.setInternationalTransferInformation().setReceivingAmount(gsmaChannelRequest.getAmount()); + + TransactionChannelRequestDTO channelRequest = new TransactionChannelRequestDTO(); // Fineract Object + Party payee = new Party(new PartyIdInfo(IdentifierType.MSISDN, transaction.getAccountNumber())); + Party payer = new Party(new PartyIdInfo(IdentifierType.MSISDN, "7543010")); + + MoneyData moneyData = new MoneyData(); + moneyData.setAmount(transaction.getAmount()); + moneyData.setCurrency(transaction.getCurrency()); + + channelRequest.setPayer(payer); + channelRequest.setPayee(payee); + channelRequest.setAmount(moneyData); + + TransactionType transactionType = new TransactionType(); + transactionType.setInitiator(PAYER); + transactionType.setInitiatorType(CONSUMER); + transactionType.setScenario(TRANSFER); + + Map extraVariables = new HashMap<>(); + extraVariables.put(IS_RTP_REQUEST, false); + extraVariables.put(TRANSACTION_TYPE, "inttransfer"); + extraVariables.put(TENANT_ID, tenantId); + + extraVariables.put(BATCH_ID, transaction.getBatchId()); + + String tenantSpecificBpmn = internationalRemittancePayer.replace("{dfspid}", tenantId); + channelRequest.setTransactionType(transactionType); + + PartyIdInfo requestedParty = (boolean)extraVariables.get(IS_RTP_REQUEST) ? channelRequest.getPayer().getPartyIdInfo() : channelRequest.getPayee().getPartyIdInfo(); + extraVariables.put(PARTY_ID_TYPE, requestedParty.getPartyIdType()); + extraVariables.put(PARTY_ID, requestedParty.getPartyIdentifier()); + + extraVariables.put(GSMA_CHANNEL_REQUEST, objectMapper.writeValueAsString(gsmaChannelRequest)); + extraVariables.put(PARTY_LOOKUP_FSPID, gsmaChannelRequest.getReceivingLei()); + extraVariables.put(INITIATOR_FSPID, gsmaChannelRequest.getRequestingLei()); + + String transactionId = zeebeProcessStarter.startZeebeWorkflow(tenantSpecificBpmn, + objectMapper.writeValueAsString(channelRequest), + extraVariables); + + System.out.println("GSMA Transaction Started with: " + transactionId); + } + + @KafkaListener(topics = "${kafka.topic.slcb.name}", groupId = "group_id") + public void listenTopicSlcb(String message) { + System.out.println("Received Message in topic SLCB and group group_id: " + message); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaConsumerConfig.java b/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaConsumerConfig.java new file mode 100644 index 00000000..5b689a39 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaConsumerConfig.java @@ -0,0 +1,40 @@ +package org.mifos.processor.bulk.kafka.config; + +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; + +import java.util.HashMap; +import java.util.Map; + +@EnableKafka +@Configuration +public class KafkaConsumerConfig { + + @Value(value = "${kafka.bootstrapAddress}") + private String bootstrapAddress; + + @Bean + public ConsumerFactory consumerFactory() { + Map props = new HashMap<>(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapAddress); + props.put(ConsumerConfig.GROUP_ID_CONFIG, "group_id"); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + return new DefaultKafkaConsumerFactory<>(props); + } + + @Bean + public ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory() { + ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(consumerFactory()); + return factory; + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaProducerConfig.java b/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaProducerConfig.java new file mode 100644 index 00000000..4ef5d8d9 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaProducerConfig.java @@ -0,0 +1,35 @@ +package org.mifos.processor.bulk.kafka.config; + +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.StringSerializer; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; + +import java.util.HashMap; +import java.util.Map; + +@Configuration +public class KafkaProducerConfig { + + @Value(value = "${kafka.bootstrapAddress}") + private String bootstrapAddress; + + @Bean + public ProducerFactory producerFactory() { + Map configProps = new HashMap<>(); + configProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapAddress); + configProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + configProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + return new DefaultKafkaProducerFactory<>(configProps); + } + + @Bean + public KafkaTemplate kafkaTemplate() { + return new KafkaTemplate<>(producerFactory()); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaTopicConfig.java b/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaTopicConfig.java new file mode 100644 index 00000000..625ddee2 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaTopicConfig.java @@ -0,0 +1,40 @@ +package org.mifos.processor.bulk.kafka.config; + +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.NewTopic; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.core.KafkaAdmin; + +import java.util.HashMap; +import java.util.Map; + +@Configuration +public class KafkaTopicConfig { + @Value(value = "${kafka.bootstrapAddress}") + private String bootstrapAddress; + + @Value(value = "${kafka.topic.gsma.name}") + private String gsmaTopicName; + + @Value(value = "${kafka.topic.slcb.name}") + private String slcbTopicName; + + @Bean + public KafkaAdmin kafkaAdmin() { + Map configs = new HashMap<>(); + configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapAddress); + return new KafkaAdmin(configs); + } + + @Bean + public NewTopic gsmaTopic() { + return new NewTopic(gsmaTopicName, 1, (short) 1); + } + + @Bean + public NewTopic slcbTopic() { + return new NewTopic(slcbTopicName, 1, (short) 1); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/schema/TransactionOlder.java b/src/main/java/org/mifos/processor/bulk/schema/TransactionOlder.java new file mode 100644 index 00000000..a783105a --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/TransactionOlder.java @@ -0,0 +1,14 @@ +package org.mifos.processor.bulk.schema; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class TransactionOlder extends Transaction { + + @JsonProperty("batchId") + private String batchId; + +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 25aadd48..76ab507c 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -7,6 +7,13 @@ camel: json-jackson: auto-discover-object-mapper: true +kafka: + bootstrapAddress: "kafka:9092" + topic: + gsma: + name: gsma + slcb: + name: slcb application: bucket-name: paymenthub-ee-dev From 261108ca08ca3fe9683562ae2be20a39746be376 Mon Sep 17 00:00:00 2001 From: Ankita Ranjan <92036361+ankita10r@users.noreply.github.com> Date: Sun, 16 Oct 2022 21:54:52 +0530 Subject: [PATCH 042/156] adding gsma p2p workflow (#40) --- .../bulk/camel/config/CamelProperties.java | 2 + .../bulk/camel/routes/InitSubBatchRoute.java | 142 +++++++++++++++++- src/main/resources/application.yaml | 3 + 3 files changed, 145 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 0bf9f462..2b516466 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -23,6 +23,8 @@ private CamelProperties() {} public static final String TRANSACTION_LIST = "transactionList"; + public static final String GSMA_CHANNEL_REQUEST = "gsmaChannelRequest"; + public static final String OVERRIDE_HEADER = "overrideHeader"; public static final String TENANT_NAME = "tenantName"; diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index a2ef5cbd..cf4fdc64 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -1,16 +1,20 @@ package org.mifos.processor.bulk.camel.routes; + + +import org.mifos.connector.common.gsma.dto.*; +import org.mifos.processor.bulk.camel.config.CamelProperties; import org.mifos.processor.bulk.schema.TransactionResult; import org.mifos.processor.bulk.utility.Utils; import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.zeebe.BpmnConfig; import org.mifos.processor.bulk.zeebe.ZeebeProcessStarter; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; - import java.util.*; - import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.camel.config.CamelProperties.GSMA_CHANNEL_REQUEST; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @Component @@ -22,6 +26,10 @@ public class InitSubBatchRoute extends BaseRouteBuilder { @Autowired private BpmnConfig bpmnConfig; + @Value("${channel.hostname}") + private String ChannelURL; + + @Override public void configure() throws Exception { @@ -61,6 +69,7 @@ public void configure() throws Exception { exchange.setProperty(ZEEBE_VARIABLE, variables); exchange.setProperty(PAYMENT_MODE, transactionList.get(0).getPaymentMode()); + }) .to("direct:start-workflow-step2"); @@ -79,6 +88,38 @@ public void configure() throws Exception { exchange.setProperty(INIT_SUB_BATCH_FAILED, false); }) .otherwise() + .when(exchangeProperty(PAYMENT_MODE).isEqualToIgnoreCase("gsma")) + .process(exchange -> { + Map variables = exchange.getProperty(ZEEBE_VARIABLE, Map.class); + variables.put(PAYMENT_MODE, "gsma"); + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + while(transactionList.size() > 0) { + GSMATransaction gsmaTransaction = convertTxnToGSMA(transactionList.get(0)); + exchange.setProperty(GSMA_CHANNEL_REQUEST, gsmaTransaction); + exchange.setProperty(INIT_SUB_BATCH_FAILED, false); + transactionList.remove(0); + } + + }) + .setHeader("Platform-TenantId", exchangeProperty(TENANT_NAME)) + .setBody(exchange-> { + GSMATransaction gsmaTransaction = exchange.getProperty(GSMA_CHANNEL_REQUEST, GSMATransaction.class); + return gsmaTransaction; + }) + .marshal().json() + .toD(ChannelURL + "/channel/gsma/transfer" + "?bridgeEndpoint=true&throwExceptionOnFailure=false") + .log("Completed start of workflow for gsma") + .choice() + .when(header("CamelHttpResponseCode").isEqualTo(200)) + .process(exchange -> { + logger.info("reached here"); + exchange.setProperty(INIT_SUB_BATCH_FAILED, false);}) + .otherwise() + .process(exchange -> { + exchange.setProperty(INIT_SUB_BATCH_FAILED, true); + }) + .endChoice() + .otherwise() .to("direct:payment-mode-missing") .setProperty(INIT_SUB_BATCH_FAILED, constant(true)) .endChoice(); @@ -105,6 +146,103 @@ public void configure() throws Exception { }) .to("direct:update-result-file") .to("direct:upload-file"); + + } + + private GSMATransaction convertTxnToGSMA(Transaction transaction) { + GSMATransaction gsmaTransaction = new GSMATransaction(); + gsmaTransaction.setAmount(transaction.getAmount()); + gsmaTransaction.setCurrency(transaction.getCurrency()); + GsmaParty payer = new GsmaParty(); + //logger.info("Payer {} {}", transaction.getPayerIdentifier(),payer[0].); + payer.setKey("msisdn"); + payer.setValue(transaction.getPayerIdentifier()); + GsmaParty payee = new GsmaParty(); + payee.setKey("msisdn"); + payee.setValue(transaction.getPayeeIdentifier()); + GsmaParty[] debitParty = new GsmaParty[1]; + GsmaParty[] creditParty = new GsmaParty[1]; + debitParty[0] = payer; + creditParty[0] = payee; + gsmaTransaction.setDebitParty(debitParty); + gsmaTransaction.setCreditParty(creditParty); + gsmaTransaction.setRequestingOrganisationTransactionReference("string"); + gsmaTransaction.setSubType("string"); + gsmaTransaction.setDescriptionText("string"); + Fee fees = new Fee(); + fees.setFeeType(transaction.getAmount()); + fees.setFeeCurrency(transaction.getCurrency()); + fees.setFeeType("string"); + Fee[] fee = new Fee[1]; + fee[0] = fees; + gsmaTransaction.setFees(fee); + gsmaTransaction.setGeoCode("37.423825,-122.082900"); + InternationalTransferInformation internationalTransferInformation = + new InternationalTransferInformation(); + internationalTransferInformation.setQuotationReference("string"); + internationalTransferInformation.setQuoteId("string"); + internationalTransferInformation.setDeliveryMethod("directtoaccount"); + internationalTransferInformation.setOriginCountry("USA"); + internationalTransferInformation.setReceivingCountry("USA"); + internationalTransferInformation.setRelationshipSender("string"); + internationalTransferInformation.setRemittancePurpose("string"); + gsmaTransaction.setInternationalTransferInformation(internationalTransferInformation); + gsmaTransaction.setOneTimeCode("string"); + IdDocument idDocument = new IdDocument(); + idDocument.setIdType("passport"); + idDocument.setIdNumber("string"); + idDocument.setIssuerCountry("USA"); + idDocument.setExpiryDate("2022-09-28T12:51:19.260+00:00"); + idDocument.setIssueDate("2022-09-28T12:51:19.260+00:00"); + idDocument.setIssuer("string"); + idDocument.setIssuerPlace("string"); + IdDocument[] idDocuments = new IdDocument[1]; + idDocuments[0] = idDocument; + PostalAddress postalAddress = new PostalAddress(); + postalAddress.setAddressLine1("string"); + postalAddress.setAddressLine2("string"); + postalAddress.setAddressLine3("string"); + postalAddress.setCity("string"); + postalAddress.setCountry("USA"); + postalAddress.setPostalCode("string"); + postalAddress.setStateProvince("string"); + SubjectName subjectName = new SubjectName(); + subjectName.setFirstName("string"); + subjectName.setLastName("string"); + subjectName.setMiddleName("string"); + subjectName.setTitle("string"); + subjectName.setNativeName("string"); + Kyc recieverKyc = new Kyc(); + recieverKyc.setBirthCountry("USA"); + recieverKyc.setDateOfBirth("2000-11-20"); + recieverKyc.setContactPhone("string"); + recieverKyc.setEmailAddress("string"); + recieverKyc.setEmployerName("string"); + recieverKyc.setGender('m'); + recieverKyc.setIdDocument(idDocuments); + recieverKyc.setNationality("USA"); + recieverKyc.setOccupation("string"); + recieverKyc.setPostalAddress(postalAddress); + recieverKyc.setSubjectName(subjectName); + Kyc senderKyc = new Kyc(); + senderKyc.setBirthCountry("USA"); + senderKyc.setDateOfBirth("2000-11-20"); + senderKyc.setContactPhone("string"); + senderKyc.setEmailAddress("string"); + senderKyc.setEmployerName("string"); + senderKyc.setGender('m'); + senderKyc.setIdDocument(idDocuments); + senderKyc.setNationality("USA"); + senderKyc.setOccupation("string"); + senderKyc.setPostalAddress(postalAddress); + senderKyc.setSubjectName(subjectName); + gsmaTransaction.setReceiverKyc(recieverKyc); + gsmaTransaction.setSenderKyc(senderKyc); + gsmaTransaction.setServicingIdentity("string"); + gsmaTransaction.setRequestDate("2022-09-28T12:51:19.260+00:00"); + + + return gsmaTransaction; } // update Transactions status to failed diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 76ab507c..386b80df 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -37,6 +37,9 @@ operations-app: batch-summary: "/api/v1/batch" batch-transaction: "/api/v1/batch/transactions" +channel: + hostname: "http://ph-ee-connector-channel" + cloud: aws: enabled: true From c847c8db8f4580260e9ce1379c9c60bf49a08541 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Wed, 19 Oct 2022 18:43:52 +0530 Subject: [PATCH 043/156] Fixes #41 Numperformating fixed for double value (#42) --- .../processor/bulk/camel/routes/FileProcessingRoute.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java index 3f17b1f0..855c173d 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java @@ -32,7 +32,7 @@ public void configure() { .id("direct:get-transaction-array") .log("Starting route direct:get-transaction-array") .process(exchange -> { - Long totalAmount = 0L; + Double totalAmount = 0.0; Long failedAmount = 0L; Long completedAmount = 0L; String filename = exchange.getProperty(LOCAL_FILE_PATH, String.class); @@ -43,7 +43,7 @@ public void configure() { while (readValues.hasNext()) { Transaction current = readValues.next(); transactionList.add(current); - totalAmount += Long.parseLong(current.getAmount()); + totalAmount += Double.parseDouble(current.getAmount()); } reader.close(); exchange.setProperty(TRANSACTION_LIST, transactionList); From 606073b1032f74d091cc757b20490d23db8a1008 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Fri, 18 Nov 2022 18:22:31 +0530 Subject: [PATCH 044/156] Security fix (#43) --- src/main/resources/application.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 386b80df..5b9ef86b 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -45,8 +45,8 @@ cloud: enabled: true s3-base-url: "https://paymenthub-ee-dev.s3.us-east-2.amazonaws.com" credentials: - access-key: ${AWS_ACCESS_KEY:AKIAX32JM37TZOJ5AKFB} - secret-key: ${AWS_SECRET_KEY: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} + access-key: ${AWS_ACCESS_KEY:access_key_from_aws} + secret-key: ${AWS_SECRET_KEY:secret_key_from_aws} region: static: us-east-2 stack: From c8bfe80eec2f426e9ca55c58181158a230712f95 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Thu, 24 Nov 2022 23:08:47 +0530 Subject: [PATCH 045/156] Added configuration for loading endpoint based on payment mode (#44) * Added configuration for loading endpoint based on payment mode * Added configuration for cucumber test --- build.gradle | 32 +++++++++++++++++++ .../bulk/camel/config/CamelProperties.java | 4 +++ .../bulk/camel/routes/InitSubBatchRoute.java | 27 +++++++++++++++- .../bulk/config/PaymentModeApiMapping.java | 12 +++++++ .../bulk/config/PaymentModeConfiguration.java | 26 +++++++++++++++ src/main/resources/application.yaml | 7 ++++ 6 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/mifos/processor/bulk/config/PaymentModeApiMapping.java create mode 100644 src/main/java/org/mifos/processor/bulk/config/PaymentModeConfiguration.java diff --git a/build.gradle b/build.gradle index 1de06a8e..5a57bad1 100644 --- a/build.gradle +++ b/build.gradle @@ -44,6 +44,12 @@ dependencies { implementation 'org.apache.camel.springboot:camel-mail-starter:3.4.0' compileOnly 'org.projectlombok:lombok:1.18.24' annotationProcessor 'org.projectlombok:lombok:1.18.24' + + testImplementation 'io.cucumber:cucumber-junit:7.8.1' + testImplementation 'org.apache.camel:camel-test:3.4.0' + testImplementation 'org.springframework.boot:spring-boot-starter-test:2.5.4' + testImplementation "com.google.truth:truth:1.1.3" + testImplementation 'com.google.code.gson:gson:2.9.0' } group = 'org.mifos' @@ -62,3 +68,29 @@ publishing { tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } + +tasks.named('test') { + useJUnitPlatform() +} + + +configurations { + cucumberRuntime { + extendsFrom testImplementation + } +} + +task cucumberCli() { + dependsOn assemble, testClasses + doLast { + javaexec { + main = "io.cucumber.core.cli.Main" + classpath = configurations.cucumberRuntime + sourceSets.main.output + sourceSets.test.output + args = [ + '--plugin', 'pretty', + '--plugin', 'html:target/cucumber-report.html', + '--glue', 'org.mifos.connector.slcb.cucumber', + 'src/test/java/resources'] + } + } +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 2b516466..7716bd0d 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -45,4 +45,8 @@ private CamelProperties() {} public static final String ZEEBE_VARIABLE = "zeebeVariable"; + public static final String EXTERNAL_ENDPOINT_FAILED = "extEndpointFailed"; + + public static final String EXTERNAL_ENDPOINT = "extEndpoint"; + } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index cf4fdc64..8abbcb6a 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -4,6 +4,8 @@ import org.mifos.connector.common.gsma.dto.*; import org.mifos.processor.bulk.camel.config.CamelProperties; +import org.mifos.processor.bulk.config.PaymentModeApiMapping; +import org.mifos.processor.bulk.config.PaymentModeConfiguration; import org.mifos.processor.bulk.schema.TransactionResult; import org.mifos.processor.bulk.utility.Utils; import org.mifos.processor.bulk.schema.Transaction; @@ -26,6 +28,9 @@ public class InitSubBatchRoute extends BaseRouteBuilder { @Autowired private BpmnConfig bpmnConfig; + @Autowired + private PaymentModeConfiguration paymentModeConfiguration; + @Value("${channel.hostname}") private String ChannelURL; @@ -107,7 +112,7 @@ public void configure() throws Exception { return gsmaTransaction; }) .marshal().json() - .toD(ChannelURL + "/channel/gsma/transfer" + "?bridgeEndpoint=true&throwExceptionOnFailure=false") + .to("direct:external-api-call") // loads the endpoint based on configuration and calls the external api .log("Completed start of workflow for gsma") .choice() .when(header("CamelHttpResponseCode").isEqualTo(200)) @@ -147,6 +152,26 @@ public void configure() throws Exception { .to("direct:update-result-file") .to("direct:upload-file"); + + from("direct:external-api-call") + .id("direct:external-api-call") + .log("Starting route direct:external-api-call") + .process(exchange -> { + String paymentMde = exchange.getProperty(PAYMENT_MODE, String.class); + PaymentModeApiMapping mapping = paymentModeConfiguration.getByMode(paymentMde); + if (mapping == null) { + exchange.setProperty(EXTERNAL_ENDPOINT_FAILED, true); + } else { + exchange.setProperty(EXTERNAL_ENDPOINT_FAILED, false); + exchange.setProperty(EXTERNAL_ENDPOINT, mapping.getEndpoint()); + } + }) + .choice() + .when(exchangeProperty(EXTERNAL_ENDPOINT_FAILED).isEqualTo(false)) + .toD(ChannelURL + "${exchangeProperty.EXTERNAL_ENDPOINT}" + "?bridgeEndpoint=true&throwExceptionOnFailure=false") + .otherwise() + .endChoice(); + } private GSMATransaction convertTxnToGSMA(Transaction transaction) { diff --git a/src/main/java/org/mifos/processor/bulk/config/PaymentModeApiMapping.java b/src/main/java/org/mifos/processor/bulk/config/PaymentModeApiMapping.java new file mode 100644 index 00000000..f7a78456 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/config/PaymentModeApiMapping.java @@ -0,0 +1,12 @@ +package org.mifos.processor.bulk.config; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +public class PaymentModeApiMapping { + + private String id, endpoint; +} diff --git a/src/main/java/org/mifos/processor/bulk/config/PaymentModeConfiguration.java b/src/main/java/org/mifos/processor/bulk/config/PaymentModeConfiguration.java new file mode 100644 index 00000000..23f3b993 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/config/PaymentModeConfiguration.java @@ -0,0 +1,26 @@ +package org.mifos.processor.bulk.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; + +@Component +@ConfigurationProperties(prefix = "bpmn") +@Setter +@Getter +public class PaymentModeConfiguration { + + private List mappings = new ArrayList<>(); + + public PaymentModeApiMapping getByMode(String paymentMode) { + return getMappings().stream() + .filter(p -> p.getId().equalsIgnoreCase(paymentMode)) + .findFirst() + .orElse(null); + } + +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 5b9ef86b..52939c96 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -75,6 +75,13 @@ bpmn: bulk-processor: "bulk_processor-ibank-usa" slcb: "slcb-{dfspid}" +payment-mode: + mapping: + - id: "GMSA" + endpoint: "/channel/gsma/transfer" + - id: "MOJALOOP" + endpoint: "/channel/transfer" + config: minimum-successful-tx-ratio: 0.90 partylookup: From 5b3026420a9328158662e5fe53865c7303072051 Mon Sep 17 00:00:00 2001 From: Ankita Ranjan <92036361+ankita10r@users.noreply.github.com> Date: Fri, 25 Nov 2022 20:08:08 +0530 Subject: [PATCH 046/156] added loop logic for starting gsma flow (#45) --- .../bulk/camel/routes/InitSubBatchRoute.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index 8abbcb6a..e54dda74 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -2,6 +2,7 @@ +import org.apache.camel.Exchange; import org.mifos.connector.common.gsma.dto.*; import org.mifos.processor.bulk.camel.config.CamelProperties; import org.mifos.processor.bulk.config.PaymentModeApiMapping; @@ -98,14 +99,18 @@ public void configure() throws Exception { Map variables = exchange.getProperty(ZEEBE_VARIABLE, Map.class); variables.put(PAYMENT_MODE, "gsma"); List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); - while(transactionList.size() > 0) { - GSMATransaction gsmaTransaction = convertTxnToGSMA(transactionList.get(0)); - exchange.setProperty(GSMA_CHANNEL_REQUEST, gsmaTransaction); - exchange.setProperty(INIT_SUB_BATCH_FAILED, false); - transactionList.remove(0); - } + exchange.setProperty("length",transactionList.size()); }) + .loop(simple("${exchangeProperty.length}")) + .process(exchange -> { + int index = exchange.getProperty(Exchange.LOOP_INDEX, Integer.class); + GSMATransaction gsmaTransaction = + convertTxnToGSMA((Transaction) exchange.getProperty(TRANSACTION_LIST, List.class) + .get(index)); + exchange.setProperty(GSMA_CHANNEL_REQUEST, gsmaTransaction); + exchange.setProperty(INIT_SUB_BATCH_FAILED, false); + }) .setHeader("Platform-TenantId", exchangeProperty(TENANT_NAME)) .setBody(exchange-> { GSMATransaction gsmaTransaction = exchange.getProperty(GSMA_CHANNEL_REQUEST, GSMATransaction.class); @@ -114,6 +119,7 @@ public void configure() throws Exception { .marshal().json() .to("direct:external-api-call") // loads the endpoint based on configuration and calls the external api .log("Completed start of workflow for gsma") + .end() .choice() .when(header("CamelHttpResponseCode").isEqualTo(200)) .process(exchange -> { From 6eb3147067b7cd06a4caccfcdde8df17f8954ffc Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Tue, 27 Dec 2022 17:16:40 +0530 Subject: [PATCH 047/156] Mojaloop and GSMA integration with api configuration (#46) * Added integration code for mojaloop payment mode * Implemented logic for bulk connector & payment connector separation using config * Added test case and used it to eliminate bugs * Removed unnecessary changes and sysout statement * Added custom exception while dto conversion * Minor buf fix + new test case --- build.gradle | 6 +- .../bulk/camel/config/CamelProperties.java | 12 + .../processor/DTOJsonConversionException.java | 8 + .../bulk/camel/processor/GsmaApiPayload.java | 31 +++ .../camel/processor/MojaloopApiPayload.java | 30 +++ .../camel/routes/FileProcessingRoute.java | 1 + .../bulk/camel/routes/InitSubBatchRoute.java | 211 +++++++----------- .../bulk/config/ExternalApiPayloadConfig.java | 47 ++++ .../bulk/config/PaymentModeConfiguration.java | 9 +- ...piMapping.java => PaymentModeMapping.java} | 4 +- .../bulk/config/PaymentModeType.java | 17 ++ .../mifos/processor/bulk/utility/Utils.java | 142 +++++++++++- src/main/resources/application-test.yaml | 9 + src/main/resources/application.yaml | 9 +- .../BulkProcessorApplicationTests.java | 4 +- .../processor/cucumber/CucumberContext.java | 29 +++ .../cucumber/stepdef/BaseStepDef.java | 40 ++++ .../stepdef/ConfigurationTestStepDef.java | 81 +++++++ .../cucumber/stepdef/InitRouteStepDef.java | 91 ++++++++ src/test/java/resources/configuration.feature | 21 ++ .../java/resources/init_route_test.feature | 31 +++ 21 files changed, 680 insertions(+), 153 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/camel/processor/DTOJsonConversionException.java create mode 100644 src/main/java/org/mifos/processor/bulk/camel/processor/GsmaApiPayload.java create mode 100644 src/main/java/org/mifos/processor/bulk/camel/processor/MojaloopApiPayload.java create mode 100644 src/main/java/org/mifos/processor/bulk/config/ExternalApiPayloadConfig.java rename src/main/java/org/mifos/processor/bulk/config/{PaymentModeApiMapping.java => PaymentModeMapping.java} (66%) create mode 100644 src/main/java/org/mifos/processor/bulk/config/PaymentModeType.java create mode 100644 src/main/resources/application-test.yaml create mode 100644 src/test/java/org/mifos/processor/cucumber/CucumberContext.java create mode 100644 src/test/java/org/mifos/processor/cucumber/stepdef/BaseStepDef.java create mode 100644 src/test/java/org/mifos/processor/cucumber/stepdef/ConfigurationTestStepDef.java create mode 100644 src/test/java/org/mifos/processor/cucumber/stepdef/InitRouteStepDef.java create mode 100644 src/test/java/resources/configuration.feature create mode 100644 src/test/java/resources/init_route_test.feature diff --git a/build.gradle b/build.gradle index 5a57bad1..5e7f7395 100644 --- a/build.gradle +++ b/build.gradle @@ -39,13 +39,15 @@ dependencies { implementation 'io.camunda:zeebe-client-java:1.1.0' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.12.0' implementation 'org.springframework.kafka:spring-kafka:2.5.8.RELEASE' - testImplementation 'org.springframework.boot:spring-boot-starter-test:2.5.2' implementation 'org.apache.camel:camel-jackson:3.4.0' implementation 'org.apache.camel.springboot:camel-mail-starter:3.4.0' compileOnly 'org.projectlombok:lombok:1.18.24' annotationProcessor 'org.projectlombok:lombok:1.18.24' + testImplementation 'org.apache.camel:camel-test-spring-junit5:3.4.0' testImplementation 'io.cucumber:cucumber-junit:7.8.1' + testImplementation 'io.cucumber:cucumber-spring:7.8.1' + testImplementation 'io.cucumber:cucumber-java:7.8.1' testImplementation 'org.apache.camel:camel-test:3.4.0' testImplementation 'org.springframework.boot:spring-boot-starter-test:2.5.4' testImplementation "com.google.truth:truth:1.1.3" @@ -89,7 +91,7 @@ task cucumberCli() { args = [ '--plugin', 'pretty', '--plugin', 'html:target/cucumber-report.html', - '--glue', 'org.mifos.connector.slcb.cucumber', + '--glue', 'org.mifos.processor.cucumber', 'src/test/java/resources'] } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 7716bd0d..19eb1267 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -23,6 +23,10 @@ private CamelProperties() {} public static final String TRANSACTION_LIST = "transactionList"; + public static final String TRANSACTION_LIST_LENGTH = "transactionListLength"; + + public static final String TRANSACTION_LIST_ELEMENT = "transactionListElement"; + public static final String GSMA_CHANNEL_REQUEST = "gsmaChannelRequest"; public static final String OVERRIDE_HEADER = "overrideHeader"; @@ -49,4 +53,12 @@ private CamelProperties() {} public static final String EXTERNAL_ENDPOINT = "extEndpoint"; + public static final String PAYLOAD_LIST = "payloadList"; + + public static final String IS_PAYMENT_MODE_VALID = "isPaymentModeValid"; + + public static final String PAYMENT_MODE_TYPE = "paymentModeType"; + + public static final String PAYLOAD = "payload"; + } diff --git a/src/main/java/org/mifos/processor/bulk/camel/processor/DTOJsonConversionException.java b/src/main/java/org/mifos/processor/bulk/camel/processor/DTOJsonConversionException.java new file mode 100644 index 00000000..954325be --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/processor/DTOJsonConversionException.java @@ -0,0 +1,8 @@ +package org.mifos.processor.bulk.camel.processor; + +public class DTOJsonConversionException extends RuntimeException { + + public DTOJsonConversionException(Class dtoClass) { + super("Unable to convert " + dtoClass.getName() + "to json string"); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/processor/GsmaApiPayload.java b/src/main/java/org/mifos/processor/bulk/camel/processor/GsmaApiPayload.java new file mode 100644 index 00000000..2bc4b699 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/processor/GsmaApiPayload.java @@ -0,0 +1,31 @@ +package org.mifos.processor.bulk.camel.processor; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.camel.Exchange; +import org.mifos.connector.common.gsma.dto.GSMATransaction; +import org.mifos.processor.bulk.schema.Transaction; +import org.mifos.processor.bulk.utility.Utils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import java.util.function.Function; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_ELEMENT; + +@Component +public class GsmaApiPayload implements Function { + + @Autowired + ObjectMapper objectMapper; + + @Override + public String apply(Exchange exchange) { + + Transaction transaction = exchange.getProperty(TRANSACTION_LIST_ELEMENT, Transaction.class); + GSMATransaction gsmaTransaction = Utils.convertTxnToGSMA(transaction); + try { + return objectMapper.writeValueAsString(gsmaTransaction); + } catch (JsonProcessingException e) { + throw new DTOJsonConversionException(GSMATransaction.class); + } + } +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/processor/MojaloopApiPayload.java b/src/main/java/org/mifos/processor/bulk/camel/processor/MojaloopApiPayload.java new file mode 100644 index 00000000..94ff7c73 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/processor/MojaloopApiPayload.java @@ -0,0 +1,30 @@ +package org.mifos.processor.bulk.camel.processor; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.camel.Exchange; +import org.mifos.connector.common.channel.dto.TransactionChannelRequestDTO; +import org.mifos.processor.bulk.schema.Transaction; +import org.mifos.processor.bulk.utility.Utils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import java.util.function.Function; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_ELEMENT; + +@Component +public class MojaloopApiPayload implements Function { + + @Autowired + ObjectMapper objectMapper; + + @Override + public String apply(Exchange exchange) { + Transaction transaction = exchange.getProperty(TRANSACTION_LIST_ELEMENT, Transaction.class); + TransactionChannelRequestDTO inboundTransferPayload = Utils.convertTxnToInboundTransferPayload(transaction); + try { + return objectMapper.writeValueAsString(inboundTransferPayload); + } catch (JsonProcessingException e) { + throw new DTOJsonConversionException(MojaloopApiPayload.class); + } + } +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java index 855c173d..6b6a5bf8 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java @@ -47,6 +47,7 @@ public void configure() { } reader.close(); exchange.setProperty(TRANSACTION_LIST, transactionList); + exchange.setProperty(TRANSACTION_LIST_LENGTH, transactionList.size()); exchange.setProperty(TOTAL_AMOUNT, totalAmount); exchange.setProperty(ONGOING_AMOUNT, totalAmount); // initially ongoing amount is same as total amount exchange.setProperty(FAILED_AMOUNT, failedAmount); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index e54dda74..d7bd74fd 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -1,12 +1,7 @@ package org.mifos.processor.bulk.camel.routes; - - import org.apache.camel.Exchange; -import org.mifos.connector.common.gsma.dto.*; -import org.mifos.processor.bulk.camel.config.CamelProperties; -import org.mifos.processor.bulk.config.PaymentModeApiMapping; -import org.mifos.processor.bulk.config.PaymentModeConfiguration; +import org.mifos.processor.bulk.config.*; import org.mifos.processor.bulk.schema.TransactionResult; import org.mifos.processor.bulk.utility.Utils; import org.mifos.processor.bulk.schema.Transaction; @@ -16,8 +11,8 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import java.util.*; +import java.util.function.Function; import static org.mifos.processor.bulk.camel.config.CamelProperties.*; -import static org.mifos.processor.bulk.camel.config.CamelProperties.GSMA_CHANNEL_REQUEST; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @Component @@ -32,6 +27,9 @@ public class InitSubBatchRoute extends BaseRouteBuilder { @Autowired private PaymentModeConfiguration paymentModeConfiguration; + @Autowired + private ExternalApiPayloadConfig externalApiPayloadConfig; + @Value("${channel.hostname}") private String ChannelURL; @@ -79,47 +77,73 @@ public void configure() throws Exception { }) .to("direct:start-workflow-step2"); - // Loops through each transaction and start the respective workflow + from("direct:start-workflow-step2") .id("direct:start-flow-step2") .log("Starting route direct:start-flow-step2") + .to("direct:validate-payment-mode") + .choice() + // if invalid payment mode + .when(exchangeProperty(IS_PAYMENT_MODE_VALID).isEqualTo(false)) + .to("direct:payment-mode-missing") + .setProperty(INIT_SUB_BATCH_FAILED, constant(true)) + // else + .otherwise() + .to("direct:start-workflow-step3") + .endChoice(); + + from("direct:start-workflow-step3") + .id("direct:start-flow-step3") + .log("Starting route direct:start-flow-step3") .choice() - .when(exchangeProperty(PAYMENT_MODE).isEqualToIgnoreCase("slcb")) + // if type of payment mode is bulk + .when(exchangeProperty(PAYMENT_MODE_TYPE).isEqualTo(PaymentModeType.BULK)) .process(exchange -> { + String paymentMode = exchange.getProperty(PAYMENT_MODE, String.class); + PaymentModeMapping mapping = paymentModeConfiguration.getByMode(paymentMode); + String tenantName = exchange.getProperty(TENANT_NAME, String.class); Map variables = exchange.getProperty(ZEEBE_VARIABLE, Map.class); - variables.put(PAYMENT_MODE, "slcb"); + variables.put(PAYMENT_MODE, paymentMode); zeebeProcessStarter.startZeebeWorkflow( - Utils.getTenantSpecificWorkflowId(bpmnConfig.slcbBpmn, tenantName), variables); + Utils.getBulkConnectorBpmnName(mapping.getEndpoint(), mapping.getId().toLowerCase(), tenantName), + variables); exchange.setProperty(INIT_SUB_BATCH_FAILED, false); }) + // if type of payment mode is payment todo // else case or else if case ? .otherwise() - .when(exchangeProperty(PAYMENT_MODE).isEqualToIgnoreCase("gsma")) - .process(exchange -> { - Map variables = exchange.getProperty(ZEEBE_VARIABLE, Map.class); - variables.put(PAYMENT_MODE, "gsma"); - List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); - exchange.setProperty("length",transactionList.size()); - - }) - .loop(simple("${exchangeProperty.length}")) + .loop(simple("${exchangeProperty." + TRANSACTION_LIST_LENGTH + "}")) .process(exchange -> { int index = exchange.getProperty(Exchange.LOOP_INDEX, Integer.class); - GSMATransaction gsmaTransaction = - convertTxnToGSMA((Transaction) exchange.getProperty(TRANSACTION_LIST, List.class) - .get(index)); - exchange.setProperty(GSMA_CHANNEL_REQUEST, gsmaTransaction); - exchange.setProperty(INIT_SUB_BATCH_FAILED, false); + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + Transaction transaction = transactionList.get(index); + exchange.setProperty(TRANSACTION_LIST_ELEMENT, transaction); }) .setHeader("Platform-TenantId", exchangeProperty(TENANT_NAME)) - .setBody(exchange-> { - GSMATransaction gsmaTransaction = exchange.getProperty(GSMA_CHANNEL_REQUEST, GSMATransaction.class); - return gsmaTransaction; - }) + .to("direct:dynamic-payload-setter") .marshal().json() - .to("direct:external-api-call") // loads the endpoint based on configuration and calls the external api - .log("Completed start of workflow for gsma") - .end() + .to("direct:external-api-call") + .to("direct:external-api-response-handler") + .end() // end loop block + .endChoice(); + + from("direct:dynamic-payload-setter") + .id("direct:runtime-payload-test") + .log("Starting route direct:runtime-payload-test") + .process(exchange -> { + String mode = exchange.getProperty(PAYMENT_MODE, String.class); + Function localPayloadVariable = externalApiPayloadConfig.getApiPayloadSetter(mode); + logger.info("MODE FOR API CALL : {}", mode); + logger.info("localPayloadVariable: {}", localPayloadVariable); + exchange.setProperty("body", localPayloadVariable.apply(exchange)); + }) + // this payload variable returns the body for respective payment modes + .setBody(simple("${exchangeProperty.body}")); + + // Loops through each transaction and start the respective workflow + from("direct:external-api-response-handler") + .id("direct:external-api-response-handler") + .log("Starting route direct:external-api-response-handler") .choice() .when(header("CamelHttpResponseCode").isEqualTo(200)) .process(exchange -> { @@ -129,10 +153,6 @@ public void configure() throws Exception { .process(exchange -> { exchange.setProperty(INIT_SUB_BATCH_FAILED, true); }) - .endChoice() - .otherwise() - .to("direct:payment-mode-missing") - .setProperty(INIT_SUB_BATCH_FAILED, constant(true)) .endChoice(); from("direct:payment-mode-missing") @@ -147,7 +167,7 @@ public void configure() throws Exception { exchange.setProperty(RESULT_TRANSACTION_LIST, transactionResultList); exchange.setProperty(RESULT_FILE, resultFile); }) - // setting localfilpath as result file to make sure result file is uploaded + // setting localfilepath as result file to make sure result file is uploaded .setProperty(LOCAL_FILE_PATH, exchangeProperty(RESULT_FILE)) .setProperty(OVERRIDE_HEADER, constant(true)) .process(exchange -> { @@ -163,117 +183,38 @@ public void configure() throws Exception { .id("direct:external-api-call") .log("Starting route direct:external-api-call") .process(exchange -> { - String paymentMde = exchange.getProperty(PAYMENT_MODE, String.class); - PaymentModeApiMapping mapping = paymentModeConfiguration.getByMode(paymentMde); + String paymentMode = exchange.getProperty(PAYMENT_MODE, String.class); + PaymentModeMapping mapping = paymentModeConfiguration.getByMode(paymentMode); if (mapping == null) { exchange.setProperty(EXTERNAL_ENDPOINT_FAILED, true); + logger.info("Failed to get the payment mode config, check the configuration for payment mode"); } else { exchange.setProperty(EXTERNAL_ENDPOINT_FAILED, false); exchange.setProperty(EXTERNAL_ENDPOINT, mapping.getEndpoint()); + logger.info("Got the config with routing to endpoint {}", mapping.getEndpoint()); } }) .choice() .when(exchangeProperty(EXTERNAL_ENDPOINT_FAILED).isEqualTo(false)) - .toD(ChannelURL + "${exchangeProperty.EXTERNAL_ENDPOINT}" + "?bridgeEndpoint=true&throwExceptionOnFailure=false") + .log("Making API call to endpoint ${exchangeProperty.extEndpoint}") + .toD(ChannelURL + "${exchangeProperty.extEndpoint}" + "?bridgeEndpoint=true&throwExceptionOnFailure=false") .otherwise() .endChoice(); - } - - private GSMATransaction convertTxnToGSMA(Transaction transaction) { - GSMATransaction gsmaTransaction = new GSMATransaction(); - gsmaTransaction.setAmount(transaction.getAmount()); - gsmaTransaction.setCurrency(transaction.getCurrency()); - GsmaParty payer = new GsmaParty(); - //logger.info("Payer {} {}", transaction.getPayerIdentifier(),payer[0].); - payer.setKey("msisdn"); - payer.setValue(transaction.getPayerIdentifier()); - GsmaParty payee = new GsmaParty(); - payee.setKey("msisdn"); - payee.setValue(transaction.getPayeeIdentifier()); - GsmaParty[] debitParty = new GsmaParty[1]; - GsmaParty[] creditParty = new GsmaParty[1]; - debitParty[0] = payer; - creditParty[0] = payee; - gsmaTransaction.setDebitParty(debitParty); - gsmaTransaction.setCreditParty(creditParty); - gsmaTransaction.setRequestingOrganisationTransactionReference("string"); - gsmaTransaction.setSubType("string"); - gsmaTransaction.setDescriptionText("string"); - Fee fees = new Fee(); - fees.setFeeType(transaction.getAmount()); - fees.setFeeCurrency(transaction.getCurrency()); - fees.setFeeType("string"); - Fee[] fee = new Fee[1]; - fee[0] = fees; - gsmaTransaction.setFees(fee); - gsmaTransaction.setGeoCode("37.423825,-122.082900"); - InternationalTransferInformation internationalTransferInformation = - new InternationalTransferInformation(); - internationalTransferInformation.setQuotationReference("string"); - internationalTransferInformation.setQuoteId("string"); - internationalTransferInformation.setDeliveryMethod("directtoaccount"); - internationalTransferInformation.setOriginCountry("USA"); - internationalTransferInformation.setReceivingCountry("USA"); - internationalTransferInformation.setRelationshipSender("string"); - internationalTransferInformation.setRemittancePurpose("string"); - gsmaTransaction.setInternationalTransferInformation(internationalTransferInformation); - gsmaTransaction.setOneTimeCode("string"); - IdDocument idDocument = new IdDocument(); - idDocument.setIdType("passport"); - idDocument.setIdNumber("string"); - idDocument.setIssuerCountry("USA"); - idDocument.setExpiryDate("2022-09-28T12:51:19.260+00:00"); - idDocument.setIssueDate("2022-09-28T12:51:19.260+00:00"); - idDocument.setIssuer("string"); - idDocument.setIssuerPlace("string"); - IdDocument[] idDocuments = new IdDocument[1]; - idDocuments[0] = idDocument; - PostalAddress postalAddress = new PostalAddress(); - postalAddress.setAddressLine1("string"); - postalAddress.setAddressLine2("string"); - postalAddress.setAddressLine3("string"); - postalAddress.setCity("string"); - postalAddress.setCountry("USA"); - postalAddress.setPostalCode("string"); - postalAddress.setStateProvince("string"); - SubjectName subjectName = new SubjectName(); - subjectName.setFirstName("string"); - subjectName.setLastName("string"); - subjectName.setMiddleName("string"); - subjectName.setTitle("string"); - subjectName.setNativeName("string"); - Kyc recieverKyc = new Kyc(); - recieverKyc.setBirthCountry("USA"); - recieverKyc.setDateOfBirth("2000-11-20"); - recieverKyc.setContactPhone("string"); - recieverKyc.setEmailAddress("string"); - recieverKyc.setEmployerName("string"); - recieverKyc.setGender('m'); - recieverKyc.setIdDocument(idDocuments); - recieverKyc.setNationality("USA"); - recieverKyc.setOccupation("string"); - recieverKyc.setPostalAddress(postalAddress); - recieverKyc.setSubjectName(subjectName); - Kyc senderKyc = new Kyc(); - senderKyc.setBirthCountry("USA"); - senderKyc.setDateOfBirth("2000-11-20"); - senderKyc.setContactPhone("string"); - senderKyc.setEmailAddress("string"); - senderKyc.setEmployerName("string"); - senderKyc.setGender('m'); - senderKyc.setIdDocument(idDocuments); - senderKyc.setNationality("USA"); - senderKyc.setOccupation("string"); - senderKyc.setPostalAddress(postalAddress); - senderKyc.setSubjectName(subjectName); - gsmaTransaction.setReceiverKyc(recieverKyc); - gsmaTransaction.setSenderKyc(senderKyc); - gsmaTransaction.setServicingIdentity("string"); - gsmaTransaction.setRequestDate("2022-09-28T12:51:19.260+00:00"); - - return gsmaTransaction; + from("direct:validate-payment-mode") + .id("direct:validate-payment-mode") + .log("Starting route direct:validate-payment-mode") + .process(exchange -> { + String paymentMde = exchange.getProperty(PAYMENT_MODE, String.class); + PaymentModeMapping mapping = paymentModeConfiguration.getByMode(paymentMde); + if (mapping == null) { + exchange.setProperty(IS_PAYMENT_MODE_VALID, false); + } else { + exchange.setProperty(IS_PAYMENT_MODE_VALID, true); + exchange.setProperty(PAYMENT_MODE_TYPE, mapping.getType()); + } + }); } // update Transactions status to failed diff --git a/src/main/java/org/mifos/processor/bulk/config/ExternalApiPayloadConfig.java b/src/main/java/org/mifos/processor/bulk/config/ExternalApiPayloadConfig.java new file mode 100644 index 00000000..a586e208 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/config/ExternalApiPayloadConfig.java @@ -0,0 +1,47 @@ +package org.mifos.processor.bulk.config; + +import lombok.Getter; +import org.apache.camel.Exchange; +import org.apache.camel.Processor; +import org.mifos.processor.bulk.camel.processor.GsmaApiPayload; +import org.mifos.processor.bulk.camel.processor.MojaloopApiPayload; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +@Configuration +@Getter +public class ExternalApiPayloadConfig { + + private Map> payloadMap = new HashMap<>(); + + @Autowired + GsmaApiPayload gsmaApiPayload; + + @Autowired + MojaloopApiPayload mojaloopApiPayload; + + @Autowired + PaymentModeConfiguration paymentModeConfiguration; + + @PostConstruct + private void registerApiProcessor() { + for (PaymentModeMapping paymentMode: paymentModeConfiguration.getMappings()) { + if (paymentMode.getId().equalsIgnoreCase("gsma")) { + payloadMap.put(paymentMode.getId(), gsmaApiPayload); + } else if (paymentMode.getId().equalsIgnoreCase("mojaloop")) { + payloadMap.put(paymentMode.getId(), mojaloopApiPayload); + } + } + } + + public Function getApiPayloadSetter(String paymentMode) { + PaymentModeMapping mapping = paymentModeConfiguration.getByMode(paymentMode); + return payloadMap.get(mapping.getId()); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/config/PaymentModeConfiguration.java b/src/main/java/org/mifos/processor/bulk/config/PaymentModeConfiguration.java index 23f3b993..97b0960a 100644 --- a/src/main/java/org/mifos/processor/bulk/config/PaymentModeConfiguration.java +++ b/src/main/java/org/mifos/processor/bulk/config/PaymentModeConfiguration.java @@ -3,20 +3,21 @@ import lombok.Getter; import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.List; -@Component -@ConfigurationProperties(prefix = "bpmn") +@Configuration +@ConfigurationProperties(prefix = "payment-mode") @Setter @Getter public class PaymentModeConfiguration { - private List mappings = new ArrayList<>(); + private List mappings = new ArrayList<>(); - public PaymentModeApiMapping getByMode(String paymentMode) { + public PaymentModeMapping getByMode(String paymentMode) { return getMappings().stream() .filter(p -> p.getId().equalsIgnoreCase(paymentMode)) .findFirst() diff --git a/src/main/java/org/mifos/processor/bulk/config/PaymentModeApiMapping.java b/src/main/java/org/mifos/processor/bulk/config/PaymentModeMapping.java similarity index 66% rename from src/main/java/org/mifos/processor/bulk/config/PaymentModeApiMapping.java rename to src/main/java/org/mifos/processor/bulk/config/PaymentModeMapping.java index f7a78456..886a8c28 100644 --- a/src/main/java/org/mifos/processor/bulk/config/PaymentModeApiMapping.java +++ b/src/main/java/org/mifos/processor/bulk/config/PaymentModeMapping.java @@ -1,12 +1,12 @@ package org.mifos.processor.bulk.config; import lombok.Getter; -import lombok.NoArgsConstructor; import lombok.Setter; @Getter @Setter -public class PaymentModeApiMapping { +public class PaymentModeMapping { private String id, endpoint; + private PaymentModeType type; } diff --git a/src/main/java/org/mifos/processor/bulk/config/PaymentModeType.java b/src/main/java/org/mifos/processor/bulk/config/PaymentModeType.java new file mode 100644 index 00000000..6fdb6b72 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/config/PaymentModeType.java @@ -0,0 +1,17 @@ +package org.mifos.processor.bulk.config; + +import lombok.Getter; + +@Getter +public enum PaymentModeType { + + PAYMENT("PAYMENT"), + BULK("BULK"); + + private String modeType; + + PaymentModeType(String modeType) { + this.modeType = modeType; + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/utility/Utils.java b/src/main/java/org/mifos/processor/bulk/utility/Utils.java index 52cd353c..0a4d7ef5 100644 --- a/src/main/java/org/mifos/processor/bulk/utility/Utils.java +++ b/src/main/java/org/mifos/processor/bulk/utility/Utils.java @@ -1,15 +1,14 @@ package org.mifos.processor.bulk.utility; +import org.mifos.connector.common.channel.dto.TransactionChannelRequestDTO; +import org.mifos.connector.common.gsma.dto.*; +import org.mifos.connector.common.mojaloop.dto.MoneyData; +import org.mifos.connector.common.mojaloop.dto.Party; +import org.mifos.connector.common.mojaloop.dto.PartyIdInfo; +import org.mifos.connector.common.mojaloop.type.IdentifierType; import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.schema.TransactionResult; - -import javax.net.ssl.*; import java.io.*; -import java.net.URL; -import java.net.URLConnection; -import java.security.KeyManagementException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.X509Certificate; public class Utils { @@ -17,6 +16,11 @@ public static String getTenantSpecificWorkflowId(String originalWorkflowName, St return originalWorkflowName.replace("{dfspid}", tenantName); } + public static String getBulkConnectorBpmnName(String originalWorkflowName, + String paymentMode, String tenantName) { + return originalWorkflowName.replace("{MODE}", paymentMode.toLowerCase()).replace("{dfspid}", tenantName); + } + public static String mergeCsvFile(String file1, String file2) { try { // create a writer for permFile @@ -90,4 +94,128 @@ public static TransactionResult mapToResultDTO(Transaction transaction) { return transactionResult; } + public static GSMATransaction convertTxnToGSMA(Transaction transaction) { + GSMATransaction gsmaTransaction = new GSMATransaction(); + gsmaTransaction.setAmount(transaction.getAmount()); + gsmaTransaction.setCurrency(transaction.getCurrency()); + GsmaParty payer = new GsmaParty(); + //logger.info("Payer {} {}", transaction.getPayerIdentifier(),payer[0].); + payer.setKey("msisdn"); + payer.setValue(transaction.getPayerIdentifier()); + GsmaParty payee = new GsmaParty(); + payee.setKey("msisdn"); + payee.setValue(transaction.getPayeeIdentifier()); + GsmaParty[] debitParty = new GsmaParty[1]; + GsmaParty[] creditParty = new GsmaParty[1]; + debitParty[0] = payer; + creditParty[0] = payee; + gsmaTransaction.setDebitParty(debitParty); + gsmaTransaction.setCreditParty(creditParty); + gsmaTransaction.setRequestingOrganisationTransactionReference("string"); + gsmaTransaction.setSubType("string"); + gsmaTransaction.setDescriptionText("string"); + Fee fees = new Fee(); + fees.setFeeType(transaction.getAmount()); + fees.setFeeCurrency(transaction.getCurrency()); + fees.setFeeType("string"); + Fee[] fee = new Fee[1]; + fee[0] = fees; + gsmaTransaction.setFees(fee); + gsmaTransaction.setGeoCode("37.423825,-122.082900"); + InternationalTransferInformation internationalTransferInformation = + new InternationalTransferInformation(); + internationalTransferInformation.setQuotationReference("string"); + internationalTransferInformation.setQuoteId("string"); + internationalTransferInformation.setDeliveryMethod("directtoaccount"); + internationalTransferInformation.setOriginCountry("USA"); + internationalTransferInformation.setReceivingCountry("USA"); + internationalTransferInformation.setRelationshipSender("string"); + internationalTransferInformation.setRemittancePurpose("string"); + gsmaTransaction.setInternationalTransferInformation(internationalTransferInformation); + gsmaTransaction.setOneTimeCode("string"); + IdDocument idDocument = new IdDocument(); + idDocument.setIdType("passport"); + idDocument.setIdNumber("string"); + idDocument.setIssuerCountry("USA"); + idDocument.setExpiryDate("2022-09-28T12:51:19.260+00:00"); + idDocument.setIssueDate("2022-09-28T12:51:19.260+00:00"); + idDocument.setIssuer("string"); + idDocument.setIssuerPlace("string"); + IdDocument[] idDocuments = new IdDocument[1]; + idDocuments[0] = idDocument; + PostalAddress postalAddress = new PostalAddress(); + postalAddress.setAddressLine1("string"); + postalAddress.setAddressLine2("string"); + postalAddress.setAddressLine3("string"); + postalAddress.setCity("string"); + postalAddress.setCountry("USA"); + postalAddress.setPostalCode("string"); + postalAddress.setStateProvince("string"); + SubjectName subjectName = new SubjectName(); + subjectName.setFirstName("string"); + subjectName.setLastName("string"); + subjectName.setMiddleName("string"); + subjectName.setTitle("string"); + subjectName.setNativeName("string"); + Kyc recieverKyc = new Kyc(); + recieverKyc.setBirthCountry("USA"); + recieverKyc.setDateOfBirth("2000-11-20"); + recieverKyc.setContactPhone("string"); + recieverKyc.setEmailAddress("string"); + recieverKyc.setEmployerName("string"); + recieverKyc.setGender('m'); + recieverKyc.setIdDocument(idDocuments); + recieverKyc.setNationality("USA"); + recieverKyc.setOccupation("string"); + recieverKyc.setPostalAddress(postalAddress); + recieverKyc.setSubjectName(subjectName); + Kyc senderKyc = new Kyc(); + senderKyc.setBirthCountry("USA"); + senderKyc.setDateOfBirth("2000-11-20"); + senderKyc.setContactPhone("string"); + senderKyc.setEmailAddress("string"); + senderKyc.setEmployerName("string"); + senderKyc.setGender('m'); + senderKyc.setIdDocument(idDocuments); + senderKyc.setNationality("USA"); + senderKyc.setOccupation("string"); + senderKyc.setPostalAddress(postalAddress); + senderKyc.setSubjectName(subjectName); + gsmaTransaction.setReceiverKyc(recieverKyc); + gsmaTransaction.setSenderKyc(senderKyc); + gsmaTransaction.setServicingIdentity("string"); + gsmaTransaction.setRequestDate("2022-09-28T12:51:19.260+00:00"); + + + return gsmaTransaction; + } + + public static TransactionChannelRequestDTO convertTxnToInboundTransferPayload(Transaction transaction) { + TransactionChannelRequestDTO requestDTO = new TransactionChannelRequestDTO(); + + requestDTO.setAmount(new MoneyData(){{ + setCurrency(transaction.getCurrency()); + setAmount(transaction.getAmount()); + }}); + + IdentifierType identifierType; + try { + identifierType = IdentifierType.valueOf(transaction.getPaymentMode().toUpperCase()); + } catch (Exception e) { + identifierType = IdentifierType.MSISDN; + } + + // PAYER SETUP + Party payerParty = new Party(new PartyIdInfo(identifierType, transaction.getPayerIdentifier())); + + // PAYEE SETUP + Party payeeParty = new Party(new PartyIdInfo(identifierType, transaction.getPayeeIdentifier())); + + requestDTO.setPayer(payerParty); + requestDTO.setPayee(payeeParty); + requestDTO.setNote(transaction.getNote()); + + return requestDTO; + } + } diff --git a/src/main/resources/application-test.yaml b/src/main/resources/application-test.yaml new file mode 100644 index 00000000..2b80f785 --- /dev/null +++ b/src/main/resources/application-test.yaml @@ -0,0 +1,9 @@ +camel: + server-port: 5002 + +kafka: + bootstrapAddress: "localhost:9092" + +zeebe: + broker: + contactpoint: "localhost:26500" diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 52939c96..f6432821 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -76,11 +76,16 @@ bpmn: slcb: "slcb-{dfspid}" payment-mode: - mapping: - - id: "GMSA" + mappings: + - id: "GSMA" + type: "PAYMENT" endpoint: "/channel/gsma/transfer" - id: "MOJALOOP" + type: "PAYMENT" endpoint: "/channel/transfer" + - id: "SLCB" + type: "BULK" + endpoint: "bulk_connector_{MODE}-{dfspid}" config: minimum-successful-tx-ratio: 0.90 diff --git a/src/test/java/org/mifos/processor/BulkProcessorApplicationTests.java b/src/test/java/org/mifos/processor/BulkProcessorApplicationTests.java index db5ed70a..bbdf1795 100644 --- a/src/test/java/org/mifos/processor/BulkProcessorApplicationTests.java +++ b/src/test/java/org/mifos/processor/BulkProcessorApplicationTests.java @@ -2,15 +2,17 @@ import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; import java.util.UUID; @SpringBootTest +@ActiveProfiles("test") class BulkProcessorApplicationTests { @Test void contextLoads() { - System.out.println(UUID.randomUUID().toString()); + System.out.println(UUID.randomUUID()); } } diff --git a/src/test/java/org/mifos/processor/cucumber/CucumberContext.java b/src/test/java/org/mifos/processor/cucumber/CucumberContext.java new file mode 100644 index 00000000..86e8eecf --- /dev/null +++ b/src/test/java/org/mifos/processor/cucumber/CucumberContext.java @@ -0,0 +1,29 @@ +package org.mifos.processor.cucumber; + +import io.cucumber.spring.CucumberContextConfiguration; +import org.apache.camel.ProducerTemplate; +import org.apache.camel.test.spring.junit5.CamelSpringBootTest; +import org.apache.camel.test.spring.junit5.UseAdviceWith; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; + +import static com.google.common.truth.Truth.assertThat; + +@CucumberContextConfiguration +@SpringBootTest +@CamelSpringBootTest +@UseAdviceWith +@ActiveProfiles("test") +public class CucumberContext { + + @Autowired + private ProducerTemplate producerTemplate; + + @Test + void contextLoads() { + assertThat(producerTemplate).isNotNull(); + } + +} diff --git a/src/test/java/org/mifos/processor/cucumber/stepdef/BaseStepDef.java b/src/test/java/org/mifos/processor/cucumber/stepdef/BaseStepDef.java new file mode 100644 index 00000000..5d00142d --- /dev/null +++ b/src/test/java/org/mifos/processor/cucumber/stepdef/BaseStepDef.java @@ -0,0 +1,40 @@ +package org.mifos.processor.cucumber.stepdef; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.camel.CamelContext; +import org.apache.camel.Exchange; +import org.apache.camel.ProducerTemplate; +import org.mifos.processor.bulk.config.ExternalApiPayloadConfig; +import org.mifos.processor.bulk.config.PaymentModeConfiguration; +import org.mifos.processor.bulk.config.PaymentModeMapping; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +// this class is the base for all the cucumber step definitions +public class BaseStepDef { + + @Autowired + ProducerTemplate template; + + @Autowired + CamelContext context; + + @Autowired + ObjectMapper objectMapper; + + @Autowired + PaymentModeConfiguration paymentModeConfiguration; + + @Autowired + ExternalApiPayloadConfig externalApiPayloadConfig; + + Logger logger = LoggerFactory.getLogger(this.getClass()); + + protected static String tenant; + protected static String paymentMode; + protected static PaymentModeMapping paymentModeMapping; + + protected static Exchange exchange; + +} diff --git a/src/test/java/org/mifos/processor/cucumber/stepdef/ConfigurationTestStepDef.java b/src/test/java/org/mifos/processor/cucumber/stepdef/ConfigurationTestStepDef.java new file mode 100644 index 00000000..3d137c94 --- /dev/null +++ b/src/test/java/org/mifos/processor/cucumber/stepdef/ConfigurationTestStepDef.java @@ -0,0 +1,81 @@ +package org.mifos.processor.cucumber.stepdef; + +import io.cucumber.java.en.And; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import org.apache.camel.Exchange; +import org.mifos.processor.bulk.config.PaymentModeMapping; +import org.mifos.processor.bulk.config.PaymentModeType; +import org.mifos.processor.bulk.utility.Utils; +import java.util.function.Function; +import static com.google.common.truth.Truth.assertThat; + +public class ConfigurationTestStepDef extends BaseStepDef { + + @Given("Application context is loaded") + public void applicationContextIsLoaded() { + assertThat(context).isNotNull(); + } + + @When("I assert the payment mode config") + public void paymentModeConfigAssert() { + assertThat(paymentModeConfiguration).isNotNull(); + } + + @Then("I should get the non empty payment modes") + public void nonEmptyPaymentModesCheck() { + assertThat(paymentModeConfiguration.getMappings()).isNotEmpty(); + } + + @And("I should be able fetch the mapping for mode {string}") + public void fetchMappingForMode(String mode) { + PaymentModeMapping mapping = paymentModeConfiguration.getByMode(mode); + assertThat(mapping).isNotNull(); + BaseStepDef.paymentModeMapping = mapping; + } + + @And("I should get enum value {} for mode {string}") + public void getEnumValueForMode(PaymentModeType modeType, String mode) { + PaymentModeMapping mapping = paymentModeConfiguration.getByMode(mode); + assertThat(mapping.getType()).isEqualTo(modeType); + } + + @When("I have payment mode {string}") + public void setPaymentMode(String paymentMode) { + BaseStepDef.paymentMode = paymentMode; + assertThat(BaseStepDef.paymentMode).isNotEmpty(); + } + + @Then("I should get the bulk connector bpmn name {string}") + public void validateBulkConnectorBpmnName(String bpmnName) { + PaymentModeMapping mapping = BaseStepDef.paymentModeMapping; + String generatedBpmnName = Utils.getBulkConnectorBpmnName(mapping.getEndpoint(), + mapping.getId(), BaseStepDef.tenant); + assertThat(bpmnName).isEqualTo(generatedBpmnName); + } + + @And("I have tenant as {string}") + public void setTenant(String tenant) { + BaseStepDef.tenant = tenant; + assertThat(BaseStepDef.tenant).isNotEmpty(); + } + + @When("I assert the external api payload config") + public void externalApiPayloadConfigAssert() { + assertThat(externalApiPayloadConfig).isNotNull(); + } + + @Then("I should get the non empty external api payload config") + public void nonEmptyExternalApiPayloadConfigCheck() { + int size = externalApiPayloadConfig.getPayloadMap().keySet().size(); + assertThat(size).isGreaterThan(0); + } + + @And("I should be able fetch the payload setter for mode {string}") + public void fetchPayloadSetterForMode(String mode) { + Function payloadSetter = externalApiPayloadConfig.getApiPayloadSetter(mode); + assertThat(payloadSetter).isNotNull(); + } + +} diff --git a/src/test/java/org/mifos/processor/cucumber/stepdef/InitRouteStepDef.java b/src/test/java/org/mifos/processor/cucumber/stepdef/InitRouteStepDef.java new file mode 100644 index 00000000..e0f67818 --- /dev/null +++ b/src/test/java/org/mifos/processor/cucumber/stepdef/InitRouteStepDef.java @@ -0,0 +1,91 @@ +package org.mifos.processor.cucumber.stepdef; + +import com.fasterxml.jackson.core.JsonProcessingException; +import io.cucumber.java.en.And; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import org.mifos.connector.common.channel.dto.TransactionChannelRequestDTO; +import org.mifos.connector.common.gsma.dto.GSMATransaction; +import org.mifos.processor.bulk.schema.Transaction; +import java.util.UUID; +import static com.google.common.truth.Truth.assertThat; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_ELEMENT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYMENT_MODE; + +public class InitRouteStepDef extends BaseStepDef { + + @Given("I can load camel context") + public void loadCamelContext() { + context.start(); + assertThat(template).isNotNull(); + assertThat(context).isNotNull(); + } + + @When("I call the payment-mode-validation route with {string} payment mode") + public void callPaymentModeValidationRoute(String paymentMode) { + exchange = template.send("direct:validate-payment-mode", exchange -> { + exchange.setProperty(PAYMENT_MODE, paymentMode); + }); + } + + @Then("I should get a non null exchange variable") + public void exchangeVariableNullCheck() { + assertThat(exchange).isNotNull(); + } + + @And("{string} exchange variable should be {string}") + public void exchangeVariableBooleanCheck(String variableKey, String variableValue) { + if (variableValue.equalsIgnoreCase("true")) { + assertThat(exchange.getProperty(variableKey, Boolean.class)).isTrue(); + } else { + assertThat(exchange.getProperty(variableKey, Boolean.class)).isFalse(); + } + } + + @When("I call the runtime-payload route with {string} payment mode") + public void callRuntimePayloadTestRoute(String paymentMode) { + exchange = template.send("direct:dynamic-payload-setter", exchange -> { + exchange.setProperty(PAYMENT_MODE, paymentMode); + exchange.setProperty(TRANSACTION_LIST_ELEMENT, new Transaction(){{ + setId(0); + setRequestId(UUID.randomUUID().toString()); + setPaymentMode(paymentMode); + setAmount("100"); + setPayerIdentifierType("MSISDN"); + setPayeeIdentifierType("MSISDN"); + setPayerIdentifier("1234567890"); + setPayeeIdentifier("0987654321"); + setCurrency("INR"); + }}); + }); + } + + @And("The body should be of GSMA parcelable") + public void gsmaBodyDeserializeCheck() { + String body = exchange.getIn().getBody(String.class); + assertThat(body).isNotNull(); + + GSMATransaction gsmaTransaction; + try { + gsmaTransaction = objectMapper.readValue(body, GSMATransaction.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + assertThat(gsmaTransaction).isNotNull(); + } + + @And("The body should be of MOJALOOP parcelable") + public void mojaloopBodyDeserializeCheck() { + String body = exchange.getIn().getBody(String.class); + assertThat(body).isNotNull(); + + TransactionChannelRequestDTO payload; + try { + payload = objectMapper.readValue(body, TransactionChannelRequestDTO.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + assertThat(payload).isNotNull(); + } +} diff --git a/src/test/java/resources/configuration.feature b/src/test/java/resources/configuration.feature new file mode 100644 index 00000000..24c16a13 --- /dev/null +++ b/src/test/java/resources/configuration.feature @@ -0,0 +1,21 @@ +Feature: configuration test + + Scenario: Payment mode config test + Given Application context is loaded + When I assert the payment mode config + Then I should get the non empty payment modes + And I should be able fetch the mapping for mode "GSMA" + And I should get enum value PAYMENT for mode "MOJALOOP" + + Scenario: Bulk connector bpmn name test + Given Application context is loaded + When I have payment mode "SLCB" + And I have tenant as "gorilla" + And I should be able fetch the mapping for mode "SLCB" + Then I should get the bulk connector bpmn name "bulk_connector_slcb-gorilla" + + Scenario: External api payload config test + Given Application context is loaded + When I assert the external api payload config + Then I should get the non empty external api payload config + And I should be able fetch the payload setter for mode "GSMA" diff --git a/src/test/java/resources/init_route_test.feature b/src/test/java/resources/init_route_test.feature new file mode 100644 index 00000000..453505ae --- /dev/null +++ b/src/test/java/resources/init_route_test.feature @@ -0,0 +1,31 @@ +Feature: init route test + + Scenario: payment mode +ve validation route test + Given I can load camel context + When I call the payment-mode-validation route with "GSMA" payment mode + Then I should get a non null exchange variable + And "isPaymentModeValid" exchange variable should be "true" + + Scenario: payment mode -ve validation route test + Given I can load camel context + When I call the payment-mode-validation route with "P2P" payment mode + Then I should get a non null exchange variable + And "isPaymentModeValid" exchange variable should be "false" + + Scenario: runtime payload test with GSMA mode + Given I can load camel context + When I call the runtime-payload route with "GSMA" payment mode + Then I should get a non null exchange variable + And The body should be of GSMA parcelable + + Scenario: runtime payload test with gsma mode + Given I can load camel context + When I call the runtime-payload route with "gsma" payment mode + Then I should get a non null exchange variable + And The body should be of GSMA parcelable + + Scenario: runtime payload test with MOJALOOP mode + Given I can load camel context + When I call the runtime-payload route with "MOJALOOP" payment mode + Then I should get a non null exchange variable + And The body should be of MOJALOOP parcelable From 3cb738c5363e0db389f9b3820fecf6592868e7ed Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Thu, 29 Dec 2022 11:24:34 +0530 Subject: [PATCH 048/156] Dynamic payload setter fixed (#47) --- .../org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index d7bd74fd..e8ffb476 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -121,7 +121,6 @@ public void configure() throws Exception { }) .setHeader("Platform-TenantId", exchangeProperty(TENANT_NAME)) .to("direct:dynamic-payload-setter") - .marshal().json() .to("direct:external-api-call") .to("direct:external-api-response-handler") .end() // end loop block From 61e86de6675e790719c3da9bd47697d4d7fe92d0 Mon Sep 17 00:00:00 2001 From: Ankita Ranjan <92036361+ankita10r@users.noreply.github.com> Date: Thu, 23 Feb 2023 23:00:06 +0530 Subject: [PATCH 049/156] Adding implementation for phased callback (#48) --- .../bulk/ConfigurationValidator.java | 20 +++++------ .../bulk/camel/routes/BatchStatusRoute.java | 12 +++---- .../camel/routes/ProcessorStartRoute.java | 31 ++++++++++++----- .../bulk/camel/routes/SendCallbackRoute.java | 33 +++++++++++++++---- .../processor/bulk/utility/PhaseUtils.java | 25 ++++++++++++++ .../processor/bulk/zeebe/ZeebeVariables.java | 12 +++++-- .../bulk/zeebe/worker/BatchStatusWorker.java | 6 ++-- .../bulk/zeebe/worker/SendCallbackWorker.java | 14 +++++--- .../bulk/zeebe/worker/WorkerConfig.java | 4 +-- src/main/resources/application.yaml | 18 +++++++--- 10 files changed, 127 insertions(+), 48 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/utility/PhaseUtils.java diff --git a/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java b/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java index 38487ced..78a98a6e 100644 --- a/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java +++ b/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java @@ -21,10 +21,10 @@ public class ConfigurationValidator { @Value("${config.ordering.field}") private String orderingField; - @Value("${config.success-threshold-check.success-threshold}") - private int successRate; + @Value("${config.completion-threshold-check.completion-threshold}") + private int completionRate; - @Value("${config.success-threshold-check.max-retry}") + @Value("${config.completion-threshold-check.max-retry}") private int maxThresholdCheckRetry; @Value("${config.formatting.standard}") @@ -38,8 +38,8 @@ private void validate() { if (workerConfig.isOrderingWorkerEnabled) { validateOrderingConfig(); } - if (workerConfig.isSuccessThresholdCheckEnabled) { - validateSuccessThresholdConfig(); + if (workerConfig.isCompletionThresholdCheckEnabled) { + validateCompletionThresholdConfig(); validateMaxRetryFromThresholdCheck(); } if (workerConfig.isFormattingWorkerEnabled) { @@ -87,13 +87,13 @@ private void validateOrderingConfig() { } // validates the success threshold related configuration - private void validateSuccessThresholdConfig() { - if (successRate <= 0 || successRate > 100) { - throw new ConfigurationValidationException("Invalid success threshold value configured (value=" + successRate + ")."); + private void validateCompletionThresholdConfig() { + if (completionRate <= 0 || completionRate > 100) { + throw new ConfigurationValidationException("Invalid completion threshold value configured (value=" + completionRate + ")."); } - if (successRate < 50) { - logger.warn("It is advised to set the success threshold greater than 50. Currently configured as {}", successRate); + if (completionRate < 50) { + logger.warn("It is advised to set the completion threshold greater than 50. Currently configured as {}", completionRate); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java index 9d170d26..05a4682e 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java @@ -13,8 +13,8 @@ @Component public class BatchStatusRoute extends BaseRouteBuilder { - @Value("${config.success-threshold-check.success-threshold}") - private int successThreshold; + @Value("${config.completion-threshold-check.completion-threshold}") + private int completionThreshold; @Override public void configure() throws Exception { @@ -65,14 +65,14 @@ public void configure() throws Exception { .process(exchange -> { BatchDTO batchSummary = exchange.getIn().getBody(BatchDTO.class); - long percentage = (batchSummary.getSuccessful()/batchSummary.getTotal())*100; + long percentage =(long)(((double)batchSummary.getSuccessful()/batchSummary.getTotal())*100); - if (percentage >= successThreshold) { + if (percentage >= completionThreshold) { logger.info("Batch success threshold reached. Expected rate: {}, Actual Rate: {}", - successThreshold, percentage); + completionThreshold, percentage); } - exchange.setProperty(SUCCESS_RATE, percentage); + exchange.setProperty(COMPLETION_RATE, percentage); }) .otherwise() diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index c39ab39b..d278414e 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -4,6 +4,7 @@ import org.apache.camel.LoggingLevel; import org.json.JSONObject; import org.mifos.processor.bulk.file.FileTransferService; +import org.mifos.processor.bulk.utility.PhaseUtils; import org.mifos.processor.bulk.utility.Utils; import org.mifos.processor.bulk.zeebe.ZeebeProcessStarter; import org.mifos.processor.bulk.zeebe.worker.WorkerConfig; @@ -16,6 +17,7 @@ import java.io.File; import java.io.FileWriter; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.UUID; @@ -43,17 +45,23 @@ public class ProcessorStartRoute extends BaseRouteBuilder { @Value("${bpmn.flows.bulk-processor}") private String workflowId; - @Value("${config.success-threshold-check.success-threshold}") - private int successThreshold; + @Value("${config.completion-threshold-check.completion-threshold}") + private int completionThreshold; - @Value("${config.success-threshold-check.max-retry}") + @Value("${config.completion-threshold-check.max-retry}") private int maxThresholdCheckRetry; - @Value("${config.success-threshold-check.delay}") + @Value("${config.completion-threshold-check.delay}") private int thresholdCheckDelay; + @Value("${callback.max-retry}") + private int maxCallbackRetry; + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + @Autowired + PhaseUtils phaseUtils; + @Override public void configure() { setup(); @@ -149,9 +157,11 @@ private void setup() { logger.info("File uploaded {}", nm); //extracting and setting callback Url - String callbackUrl = exchange.getIn().getHeader("Callback-Url", String.class); + String callbackUrl = exchange.getIn().getHeader("X-Callback-URL", String.class); exchange.setProperty(CALLBACK_URL,callbackUrl); + List phases = phaseUtils.getValues(); + logger.info(phases.toString()); Map variables = new HashMap<>(); variables.put(BATCH_ID, batchId); variables.put(FILE_NAME, fileName); @@ -159,12 +169,16 @@ private void setup() { variables.put(PURPOSE, purpose); variables.put(TENANT_ID, exchange.getProperty(TENANT_NAME)); variables.put(CALLBACK_URL,callbackUrl); + variables.put(PHASES,phases); + variables.put(PHASE_COUNT,phases.size()); setConfigProperties(variables); JSONObject response = new JSONObject(); try { - String txnId = zeebeProcessStarter.startZeebeWorkflow(workflowId, "", variables); + String tenantSpecificWorkflowId = workflowId.replace("{dfspid}", + exchange.getProperty(TENANT_NAME).toString()); + String txnId = zeebeProcessStarter.startZeebeWorkflow(tenantSpecificWorkflowId, "", variables); if (txnId == null || txnId.isEmpty()) { response.put("errorCode", 500); response.put("errorDescription", "Unable to start zeebe workflow"); @@ -205,13 +219,14 @@ private Map setConfigProperties(Map variables) { variables.put(ORDERING_ENABLED, workerConfig.isOrderingWorkerEnabled); variables.put(SPLITTING_ENABLED, workerConfig.isSplittingWorkerEnabled); variables.put(FORMATTING_ENABLED, workerConfig.isFormattingWorkerEnabled); - variables.put(SUCCESS_THRESHOLD_CHECK_ENABLED, workerConfig.isSuccessThresholdCheckEnabled); + variables.put(COMPLETION_THRESHOLD_CHECK_ENABLED, workerConfig.isCompletionThresholdCheckEnabled); variables.put(MERGE_ENABLED, workerConfig.isMergeBackWorkerEnabled); variables.put(MAX_STATUS_RETRY, maxThresholdCheckRetry); - variables.put(SUCCESS_THRESHOLD, successThreshold); + variables.put(COMPLETION_THRESHOLD, completionThreshold); variables.put(THRESHOLD_DELAY, Utils.getZeebeTimerValue(thresholdCheckDelay)); variables.put(BULK_NOTIF_SUCCESS,false); variables.put(BULK_NOTIF_FAILURE,false); + variables.put(MAX_CALLBACK_RETRY,maxCallbackRetry); return variables; } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java index c2c25af6..c87e81b2 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java @@ -2,12 +2,11 @@ import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; -import org.apache.camel.model.dataformat.JsonLibrary; -import org.apache.camel.spi.AsEndpointUri; -import org.mifos.processor.bulk.schema.BatchDTO; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import java.util.List; + import static org.mifos.processor.bulk.camel.config.CamelProperties.*; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @@ -35,8 +34,9 @@ public void configure() throws Exception { .log("Starting route " + RouteId.SEND_CALLBACK.name()) .log("Sending callback for Batch Processing") .setBody(exchange -> { - String body = "Batch Aggregation API was complete with %: " + - exchange.getProperty(SUCCESS_RATE); + String body = String.format + ("The Batch Aggregation API was complete with : %s" + , exchange.getProperty(COMPLETION_RATE).toString()); callbackUrl = exchange.getProperty(CALLBACK_URL).toString(); return body; }) @@ -45,22 +45,31 @@ public void configure() throws Exception { .when(header("CamelHttpResponseCode").startsWith("2")) .log(LoggingLevel.INFO, "Callback sending was successful") .process(exchange -> { + List phases = (List) exchange.getProperty(PHASES); + exchange.setProperty(CALLBACK_RESPONSE_CODE, exchange.getIn() .getHeader(Exchange.HTTP_RESPONSE_CODE)); exchange.setProperty(CALLBACK_RETRY, 1); exchange.setProperty(CALLBACK_SUCCESS, true); + eliminatePhases(exchange,phases, (Integer)exchange.getProperty(PHASE_COUNT), + (Integer)exchange.getProperty(COMPLETION_RATE)); }) .otherwise() .log(LoggingLevel.ERROR, "Callback request was unsuccessful") .process(exchange -> { - if (exchange.getProperty(CALLBACK_RETRY).equals(MAX_STATUS_RETRY)) { + if (exchange.getProperty(CALLBACK_RETRY).equals(exchange.getProperty(MAX_CALLBACK_RETRY))) { + List phases = (List) exchange.getProperty(PHASES); logger.info("Retry Exhausted, setting Callback as Failed"); + eliminatePhases(exchange,phases, (Integer)exchange.getProperty(PHASE_COUNT), + (Integer)exchange.getProperty(COMPLETION_RATE)); + exchange.setProperty(CALLBACK_RESPONSE_CODE, exchange.getIn() + .getHeader(Exchange.HTTP_RESPONSE_CODE)); } else { int retry = (int) exchange.getProperty(CALLBACK_RETRY); retry++; logger.info("Retry Left {}, Setting Callback as Failed and Retrying...", - ((int) exchange.getProperty(MAX_STATUS_RETRY) - retry)); + ((int) exchange.getProperty(MAX_CALLBACK_RETRY) - retry)); exchange.setProperty(CALLBACK_RETRY, retry); exchange.setProperty(CALLBACK_RESPONSE_CODE, exchange.getIn() .getHeader(Exchange.HTTP_RESPONSE_CODE)); @@ -76,5 +85,15 @@ public void configure() throws Exception { .log("Reached Simulation"); } + public void eliminatePhases(Exchange exchange, List phases, int phaseCount, int completionRate){ + int i = 0; + while(phases.size() > 0 && phases.size() > i){ + if(phases.get(i) <= completionRate) + phases.remove(i); + i++; + } + exchange.setProperty(PHASES,phases); + exchange.setProperty(PHASE_COUNT,phaseCount); + } } diff --git a/src/main/java/org/mifos/processor/bulk/utility/PhaseUtils.java b/src/main/java/org/mifos/processor/bulk/utility/PhaseUtils.java new file mode 100644 index 00000000..e711ab23 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/utility/PhaseUtils.java @@ -0,0 +1,25 @@ +package org.mifos.processor.bulk.utility; + + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +@ConfigurationProperties(prefix = "callback_phases") +public class PhaseUtils { + + private List values; + + public List getValues() { + return values; + } + + public void setValues(List values) { + this.values = values; + + } + + +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 07312f58..2bdf2d17 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -55,7 +55,7 @@ private ZeebeVariables() { public static final String FORMATTING_ENABLED = "formattingEnabled"; - public static final String SUCCESS_THRESHOLD_CHECK_ENABLED = "successThresholdCheckEnabled"; + public static final String COMPLETION_THRESHOLD_CHECK_ENABLED = "completionThresholdCheckEnabled"; public static final String MERGE_ENABLED = "mergeEnabled"; @@ -89,9 +89,9 @@ private ZeebeVariables() { public static final String CALLBACK_RETRY = "callbackRetryCount"; - public static final String SUCCESS_THRESHOLD = "successThreshold"; + public static final String COMPLETION_THRESHOLD = "completionThreshold"; - public static final String SUCCESS_RATE = "successRate"; + public static final String COMPLETION_RATE = "completionRate"; public static final String ERROR_CODE = "errorCode"; @@ -105,7 +105,13 @@ private ZeebeVariables() { public static final String CALLBACK_URL = "callbackUrl"; + public static final String MAX_CALLBACK_RETRY = "maxCallbackRetry"; + public static final String BULK_NOTIF_SUCCESS = "isNotificationsSuccessEnabled"; public static final String BULK_NOTIF_FAILURE = "isNotificationsFailureEnabled"; + + public static final String PHASES = "phases"; + + public static final String PHASE_COUNT = "phaseCount"; } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java index 4945de5c..865f4efe 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java @@ -17,7 +17,7 @@ public void setup() { Map variables = job.getVariablesAsMap(); int retry = (int) variables.getOrDefault(RETRY, 0); - int successRate = (int) variables.getOrDefault(SUCCESS_RATE, 0); + int successRate = (int) variables.getOrDefault(COMPLETION_RATE, 0); Exchange exchange = new DefaultExchange(camelContext); exchange.setProperty(BATCH_ID, variables.get(BATCH_ID)); @@ -27,14 +27,14 @@ public void setup() { Boolean batchStatusFailed = exchange.getProperty(BATCH_STATUS_FAILED, Boolean.class); if (batchStatusFailed == null || !batchStatusFailed) { - successRate = exchange.getProperty(SUCCESS_RATE, Long.class).intValue(); + successRate = exchange.getProperty(COMPLETION_RATE, Long.class).intValue(); } else { variables.put(ERROR_CODE, exchange.getProperty(ERROR_CODE)); variables.put(ERROR_DESCRIPTION, exchange.getProperty(ERROR_DESCRIPTION)); logger.info("Error: {}, {}", variables.get(ERROR_CODE), variables.get(ERROR_DESCRIPTION)); } - variables.put(SUCCESS_RATE, successRate); + variables.put(COMPLETION_RATE, successRate); variables.put(RETRY, ++retry); logger.info("Retry: {} and Success Rate: {}", retry, successRate); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java index 58b8f578..bbb5ac60 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java @@ -19,13 +19,15 @@ public void setup() { newWorker(Worker.SEND_CALLBACK, (client, job) -> { Map variables = job.getVariablesAsMap(); - int retry = (int) variables.getOrDefault(CALLBACK_RETRY, 0); - + int retry = variables.getOrDefault(CALLBACK_RETRY, 0).equals(variables.get(MAX_STATUS_RETRY))?0: + (int) variables.getOrDefault(CALLBACK_RETRY, 0); Exchange exchange = new DefaultExchange(camelContext); - exchange.setProperty(MAX_STATUS_RETRY,variables.get(MAX_STATUS_RETRY)); + exchange.setProperty(MAX_CALLBACK_RETRY,variables.get(MAX_CALLBACK_RETRY)); exchange.setProperty(CALLBACK_RETRY,retry); exchange.setProperty(CALLBACK_URL,variables.get(CALLBACK_URL)); - exchange.setProperty(SUCCESS_RATE,variables.get(SUCCESS_RATE)); + exchange.setProperty(COMPLETION_RATE,variables.get(COMPLETION_RATE)); + exchange.setProperty(PHASES,variables.get(PHASES)); + exchange.setProperty(PHASE_COUNT,variables.get(PHASE_COUNT)); sendToCamelRoute(RouteId.SEND_CALLBACK, exchange); Boolean callbackSuccess = exchange.getProperty(CALLBACK_SUCCESS, Boolean.class); @@ -39,8 +41,10 @@ public void setup() { variables.put(CALLBACK_RETRY,exchange.getProperty(CALLBACK_RETRY)); variables.put(CALLBACK_RESPONSE_CODE,exchange.getProperty(CALLBACK_RESPONSE_CODE)); + variables.put(PHASE_COUNT,exchange.getProperty(PHASE_COUNT)); + variables.put(PHASES,exchange.getProperty(PHASES)); - logger.info("Retry: {} and Response Code {}", exchange.getProperty(CALLBACK_RETRY), exchange.getProperty(CALLBACK_RESPONSE_CODE)); + logger.debug("Retry: {} and Response Code {}", exchange.getProperty(CALLBACK_RETRY), exchange.getProperty(CALLBACK_RESPONSE_CODE)); client.newCompleteCommand(job.getKey()).variables(variables).send(); }); } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java index 1fc0de7a..50ac4c58 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java @@ -24,7 +24,7 @@ public class WorkerConfig { @Value("${config.mergeback.enable}") public boolean isMergeBackWorkerEnabled; - @Value("${config.success-threshold-check.enable}") - public boolean isSuccessThresholdCheckEnabled; + @Value("${config.completion-threshold-check.enable}") + public boolean isCompletionThresholdCheckEnabled; } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index f6432821..056f00ab 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -72,7 +72,7 @@ bpmn: international-remittance-payee: "international_remittance_payee_process-{dfspid}" international-remittance-payer: "international_remittance_payer_process-{dfspid}" debit-party-process: "debit_party_process-{dfspid}" - bulk-processor: "bulk_processor-ibank-usa" + bulk-processor: "bulk_processor-{dfspid}" slcb: "slcb-{dfspid}" payment-mode: @@ -106,11 +106,21 @@ config: enable: false backpressure: enable: false - success-threshold-check: + completion-threshold-check: enable: false - success-threshold: 95 # in percentage - max-retry: 4 + completion-threshold: 95 # in percentage + max-retry: 4 #can be as high as 30 delay: 60 # in seconds callback: + max-retry: 3 url: "http://httpstat.us/503" + + +callback_phases: + values: + - 20 + - 40 + - 60 + - 80 + - 100 \ No newline at end of file From 5e52eb185e686414163fb259ba10238d9feda6bb Mon Sep 17 00:00:00 2001 From: Ankita Ranjan <92036361+ankita10r@users.noreply.github.com> Date: Tue, 28 Feb 2023 14:49:01 +0530 Subject: [PATCH 050/156] Minor bug fix for variable name (#49) --- src/main/java/org/mifos/processor/bulk/utility/PhaseUtils.java | 2 +- src/main/resources/application.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/utility/PhaseUtils.java b/src/main/java/org/mifos/processor/bulk/utility/PhaseUtils.java index e711ab23..4b24a00f 100644 --- a/src/main/java/org/mifos/processor/bulk/utility/PhaseUtils.java +++ b/src/main/java/org/mifos/processor/bulk/utility/PhaseUtils.java @@ -7,7 +7,7 @@ import java.util.List; @Component -@ConfigurationProperties(prefix = "callback_phases") +@ConfigurationProperties(prefix = "callback-phases") public class PhaseUtils { private List values; diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 056f00ab..9a8cab3c 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -117,7 +117,7 @@ callback: url: "http://httpstat.us/503" -callback_phases: +callback-phases: values: - 20 - 40 From 10604f248929efaf134f921b5b7370a67f375822 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Thu, 9 Mar 2023 20:43:12 +0530 Subject: [PATCH 051/156] FIXES-223 addresses 222 Updated the external api call headers (#50) * FIXES-223 addresses 222 Updated the external api call headers * Log level changed to debug * Log level changed to debug --- .../mifos/processor/bulk/camel/config/CamelProperties.java | 2 ++ .../processor/bulk/camel/routes/InitSubBatchRoute.java | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 19eb1267..4201adc3 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -61,4 +61,6 @@ private CamelProperties() {} public static final String PAYLOAD = "payload"; + public static final String BATCH_ID_HEADER = "X-BatchID"; + } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index e8ffb476..e8639822 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -1,6 +1,7 @@ package org.mifos.processor.bulk.camel.routes; import org.apache.camel.Exchange; +import org.apache.camel.LoggingLevel; import org.mifos.processor.bulk.config.*; import org.mifos.processor.bulk.schema.TransactionResult; import org.mifos.processor.bulk.utility.Utils; @@ -195,8 +196,11 @@ public void configure() throws Exception { }) .choice() .when(exchangeProperty(EXTERNAL_ENDPOINT_FAILED).isEqualTo(false)) - .log("Making API call to endpoint ${exchangeProperty.extEndpoint}") + .log(LoggingLevel.DEBUG, "Making API call to endpoint ${exchangeProperty.extEndpoint} and body: ${body}") + .setHeader(Exchange.CONTENT_TYPE, constant("application/json")) + .setHeader(BATCH_ID_HEADER, simple("${exchangeProperty." + BATCH_ID + "}")) .toD(ChannelURL + "${exchangeProperty.extEndpoint}" + "?bridgeEndpoint=true&throwExceptionOnFailure=false") + .log(LoggingLevel.DEBUG, "Response body: ${body}") .otherwise() .endChoice(); From c0d14d0173471a81e5f1a90b6f13f503d946ac59 Mon Sep 17 00:00:00 2001 From: Ankita Ranjan <92036361+ankita10r@users.noreply.github.com> Date: Fri, 14 Apr 2023 19:46:13 +0530 Subject: [PATCH 052/156] Suggested Callback Time that is Resolved by Polling (#51) --- .../camel/routes/ProcessorStartRoute.java | 75 +++++++++++++------ src/main/resources/application.yaml | 2 + 2 files changed, 56 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index d278414e..3c342cda 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -57,6 +57,9 @@ public class ProcessorStartRoute extends BaseRouteBuilder { @Value("${callback.max-retry}") private int maxCallbackRetry; + @Value("${pollingApi.timer}") + private String pollApiTimer; + private final Logger logger = LoggerFactory.getLogger(this.getClass()); @Autowired @@ -73,26 +76,14 @@ private void setup() { .id("rest:POST:/batchtransactions") .log("Starting route rest:POST:/batchtransactions") .unmarshal().mimeMultipart("multipart/*") - .to("direct:validate-tenant") .process(exchange -> { - String filename = exchange.getIn().getHeader("filename", String.class); - String requestId = exchange.getIn().getHeader("X-CorrelationID", String.class); - String purpose = exchange.getIn().getHeader("Purpose", String.class); - String type = exchange.getIn().getHeader("Type", String.class); - exchange.setProperty(FILE_NAME, filename); - exchange.setProperty(REQUEST_ID, requestId); - exchange.setProperty(PURPOSE, purpose); - exchange.setProperty(BATCH_REQUEST_TYPE, type); + String batchId = UUID.randomUUID().toString(); + exchange.setProperty(BATCH_ID,batchId); + }) - .choice() - .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("raw")) - .to("direct:start-batch-process-raw") - .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("csv")) - .unmarshal().mimeMultipart("multipart/*") - .to("direct:start-batch-process-csv") - .otherwise() - .setBody(exchange -> getUnsupportedTypeJson(exchange.getProperty(BATCH_REQUEST_TYPE, String.class)).toString()) - .log("Completed execution of route rest:POST:/batchtransactions"); + .wireTap("direct:executeBatch") + .to("direct:pollingOutput"); + from("rest:POST:/bulk/transfer/{requestId}/{fileName}") @@ -102,12 +93,14 @@ private void setup() { String fileName = System.currentTimeMillis() + "_" + exchange.getIn().getHeader("fileName", String.class); String requestId = exchange.getIn().getHeader("requestId", String.class); String purpose = exchange.getIn().getHeader("purpose", String.class); - + String batchId = UUID.randomUUID().toString(); + exchange.setProperty(BATCH_ID,batchId); exchange.setProperty(FILE_NAME, fileName); exchange.setProperty(REQUEST_ID, requestId); exchange.setProperty(PURPOSE, purpose); }) - .to("direct:start-batch-process-csv"); + .wireTap("direct:start-batch-process-csv") + .to("direct:pollingOutput"); from("direct:validate-tenant") .id("direct:validate-tenant") @@ -130,7 +123,7 @@ private void setup() { String fileName = exchange.getProperty(FILE_NAME, String.class); String requestId = exchange.getProperty(REQUEST_ID, String.class); String purpose = exchange.getProperty(PURPOSE, String.class); - String batchId = UUID.randomUUID().toString(); + String batchId = exchange.getProperty(BATCH_ID,String.class); if (purpose == null || purpose.isEmpty()) { @@ -211,6 +204,46 @@ private void setup() { exchange.getIn().setBody(response.toString()); }) .log("Completed route direct:start-batch-process-raw"); + + from("direct:executeBatch") + .id("direct:executeBatch") + .log("Starting route direct:executeBatch") + .to("direct:validate-tenant") + .process(exchange -> { + String filename = exchange.getIn().getHeader("filename", String.class); + String requestId = exchange.getIn().getHeader("X-CorrelationID", String.class); + String purpose = exchange.getIn().getHeader("Purpose", String.class); + String type = exchange.getIn().getHeader("Type", String.class); + exchange.setProperty(FILE_NAME, filename); + exchange.setProperty(REQUEST_ID, requestId); + exchange.setProperty(PURPOSE, purpose); + exchange.setProperty(BATCH_REQUEST_TYPE, type); + }) + .choice() + .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("raw")) + .to("direct:start-batch-process-raw") + .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("csv")) + .unmarshal().mimeMultipart("multipart/*") + .to("direct:start-batch-process-csv") + .otherwise() + .setBody(exchange -> getUnsupportedTypeJson(exchange.getProperty(BATCH_REQUEST_TYPE, String.class)).toString()) + .log("Completed execution of route rest:POST:/batchtransactions"); + + from("direct:pollingOutput") + .id("direct:pollingOutput") + .log("Started pollingOutput route") + .process(exchange -> { +// String response = "PollingPath: /batch/Summary/{batchId} and SuggestedCallbackSeconds: # seconds"; +// response = response.replace("#",pollApiTimer).replace("{batchId}", (CharSequence) exchange.getProperty(BATCH_ID)); +// ObjectMapper mapper = new ObjectMapper(); + JSONObject json = new JSONObject(); + json.put("PollingPath", "/batch/Summary/"+exchange.getProperty(BATCH_ID)); + json.put("SuggestedCallbackSeconds",pollApiTimer); + exchange.getIn().setBody(json.toString()); + + + }); + } private Map setConfigProperties(Map variables) { diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 9a8cab3c..6040a243 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -116,6 +116,8 @@ callback: max-retry: 3 url: "http://httpstat.us/503" +pollingApi: + timer: "120" callback-phases: values: From 4b1ae42ed48454073bbe90dc1ab5a0731b0de8bf Mon Sep 17 00:00:00 2001 From: Dhruv Sonagara <78945411+dhruvsonagara@users.noreply.github.com> Date: Fri, 12 May 2023 12:40:27 +0530 Subject: [PATCH 053/156] Upgraded java version (#62) --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 5e7f7395..cba60179 100644 --- a/build.gradle +++ b/build.gradle @@ -57,7 +57,7 @@ dependencies { group = 'org.mifos' version = '0.0.1-SNAPSHOT' description = 'ph-ee-processor-bulk' -sourceCompatibility = '1.8' +sourceCompatibility = '17' publishing { publications { From 7f52ea393f3bc9745825cc8bb4967cc375b9b3db Mon Sep 17 00:00:00 2001 From: Ishan Ranasingh <44061541+truthfool@users.noreply.github.com> Date: Wed, 24 May 2023 16:42:37 +0530 Subject: [PATCH 054/156] Added loggers for workers (#56) * added loggers for workers * added debug logs * added filter for tenant --- .../processor/bulk/api/ApiOriginFilter.java | 41 +++++++++++++++++++ .../bulk/zeebe/worker/ApprovalWorker.java | 1 + .../bulk/zeebe/worker/BatchStatusWorker.java | 1 + .../bulk/zeebe/worker/FormattingWorker.java | 1 + .../bulk/zeebe/worker/InitSubBatchWorker.java | 1 + .../bulk/zeebe/worker/MergeBackWorker.java | 1 + .../bulk/zeebe/worker/OrderingWorker.java | 1 + .../bulk/zeebe/worker/PartyLookupWorker.java | 1 + .../bulk/zeebe/worker/SendCallbackWorker.java | 1 + .../bulk/zeebe/worker/SplittingWorker.java | 1 + 10 files changed, 50 insertions(+) create mode 100644 src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java diff --git a/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java b/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java new file mode 100644 index 00000000..2f849267 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java @@ -0,0 +1,41 @@ +package org.mifos.processor.bulk.api; + +import com.google.common.base.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; + +@Component +public class ApiOriginFilter implements Filter { + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { + HttpServletRequest req = (HttpServletRequest) request; + String tenant = req.getHeader("Platform-TenantId"); + Optional tenantVal = Optional.of(tenant); + if (tenantVal.isPresent()) { + logger.info("Tenant Name is : {}", tenant); + } + + } + + @Override + public void destroy() { + + } +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/ApprovalWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/ApprovalWorker.java index 8171904b..36585159 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/ApprovalWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/ApprovalWorker.java @@ -11,6 +11,7 @@ public class ApprovalWorker extends BaseWorker { @Override public void setup() { newWorker(Worker.APPROVAL, (client, job) -> { + logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); Map variables = job.getVariablesAsMap(); if (workerConfig.isApprovalWorkerEnabled) { diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java index 865f4efe..a1a8196d 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java @@ -14,6 +14,7 @@ public class BatchStatusWorker extends BaseWorker { @Override public void setup() { newWorker(Worker.BATCH_STATUS, (client, job) -> { + logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); Map variables = job.getVariablesAsMap(); int retry = (int) variables.getOrDefault(RETRY, 0); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/FormattingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/FormattingWorker.java index d2ded17b..cde6a102 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/FormattingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/FormattingWorker.java @@ -23,6 +23,7 @@ public void setup() { * 4. Uploads the updated file in cloud */ newWorker(Worker.FORMATTING, (client, job) -> { + logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); Map variables = job.getVariablesAsMap(); if (workerConfig.isFormattingWorkerEnabled) { variables.put(FORMATTING_FAILED, false); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java index ed265011..c1c75294 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java @@ -25,6 +25,7 @@ public void setup() { * 3. Initiates workflow based on the payment_mode */ newWorker(Worker.INIT_SUB_BATCH, (client, job) -> { + logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); Map variables = job.getVariablesAsMap(); List subBatches = (List) variables.get(SUB_BATCHES); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java index 3271fbaa..cf74564c 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java @@ -17,6 +17,7 @@ public class MergeBackWorker extends BaseWorker { @Override public void setup() { newWorker(Worker.MERGE_BACK, (client, job) -> { + logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); Map variables = job.getVariablesAsMap(); if (workerConfig.isMergeBackWorkerEnabled) { variables.put(MERGE_FAILED, false); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java index b75c0f54..4e8e932b 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java @@ -24,6 +24,7 @@ public void setup() { * 4. Uploads the updated file in cloud */ newWorker(Worker.ORDERING, (client, job) -> { + logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); Map variables = job.getVariablesAsMap(); if (workerConfig.isOrderingWorkerEnabled) { variables.put(ORDERING_FAILED, false); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/PartyLookupWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/PartyLookupWorker.java index e8d4b8a5..9dc01ec0 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/PartyLookupWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/PartyLookupWorker.java @@ -13,6 +13,7 @@ public class PartyLookupWorker extends BaseWorker { @Override public void setup() { newWorker(Worker.PARTY_LOOKUP, (client, job) -> { + logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); Map variables = job.getVariablesAsMap(); if (workerConfig.isPartyLookUpWorkerEnabled) { diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java index bbb5ac60..1fc0c912 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java @@ -17,6 +17,7 @@ public class SendCallbackWorker extends BaseWorker { @Override public void setup() { newWorker(Worker.SEND_CALLBACK, (client, job) -> { + logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); Map variables = job.getVariablesAsMap(); int retry = variables.getOrDefault(CALLBACK_RETRY, 0).equals(variables.get(MAX_STATUS_RETRY))?0: diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java index 8997cd5c..ab1bfb36 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java @@ -25,6 +25,7 @@ public void setup() { * 4. Sets zeebeVariable [SPLITTING_FAILED, SUB_BATCHES, SUB_BATCH_CREATED] */ newWorker(Worker.SPLITTING, (client, job) -> { + logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); Map variables = job.getVariablesAsMap(); if (workerConfig.isSplittingWorkerEnabled) { variables.put(SPLITTING_FAILED, false); From afa4c7404307ffa48cc2a1a7909e8c6bcad17b11 Mon Sep 17 00:00:00 2001 From: avikganguly01 Date: Sun, 28 May 2023 22:12:54 +0530 Subject: [PATCH 055/156] CircleCI ECR Push --- .circleci/config.yml | 48 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..42f24f81 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,48 @@ +version: 2.1 +orbs: + slack: circleci/slack@4.12.5 + aws-ecr: circleci/aws-ecr@8.2.1 +jobs: + build: + docker: + - image: cimg/openjdk:17.0.0 + - image: docker:17.05.0-ce-git + working_directory: ~/repo + environment: + # Customize the JVM maximum heap limit + JVM_OPTS: -Xmx512m + TERM: dumb + steps: + - checkout + - setup_remote_docker + +# - slack/notify: +# event: fail +# mentions: '@here' +# template: basic_fail_1 +# - slack/notify: +# event: pass +# template: basic_success_1 + + # - run: ./gradlew cucumberCli + # run tests! Slack Success/Fail Notification Step + #- run: ./gradlew test +workflows: + build: + jobs: + - aws-ecr/build-and-push-image: + aws-access-key-id: AWS_ACCESS_KEY_ID + aws-secret-access-key: AWS_SECRET_ACCESS_KEY + context: aws + extra-build-args: '--compress' + push-image: true + region: ap-south-1 + registry-id: AWS_REGISTRY_ID + repo: phee-bulk-processor + repo-scan-on-push: true + role-arn: arn:aws:iam::419830066942:role/CustomAdmin + tag: latest + - build: + context: + - slack + - aws From 92c7ce7375e43bc5a3f88ee9cbd80b7c681ee4a5 Mon Sep 17 00:00:00 2001 From: avikganguly01 Date: Sun, 28 May 2023 22:51:14 +0530 Subject: [PATCH 056/156] Docker Java 17 --- .circleci/config.yml | 5 ++--- Dockerfile | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 42f24f81..1d1fea36 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -28,12 +28,12 @@ jobs: # run tests! Slack Success/Fail Notification Step #- run: ./gradlew test workflows: - build: + build_and_push_image: jobs: - aws-ecr/build-and-push-image: aws-access-key-id: AWS_ACCESS_KEY_ID aws-secret-access-key: AWS_SECRET_ACCESS_KEY - context: aws + context: AWS extra-build-args: '--compress' push-image: true region: ap-south-1 @@ -45,4 +45,3 @@ workflows: - build: context: - slack - - aws diff --git a/Dockerfile b/Dockerfile index 67ec2e5a..4bad37e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM openjdk:13 +FROM openjdk:17 EXPOSE 5000 COPY build/libs/*.jar . From 23cbeaa1f3aca45be46767a6367853a59eb50e8d Mon Sep 17 00:00:00 2001 From: avikganguly01 Date: Sun, 28 May 2023 23:08:27 +0530 Subject: [PATCH 057/156] Gradle build before Docker build --- .circleci/config.yml | 8 ++++---- Dockerfile | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d1fea36..6f7e212f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,13 +23,16 @@ jobs: # - slack/notify: # event: pass # template: basic_success_1 - + - run: ./gradlew build # - run: ./gradlew cucumberCli # run tests! Slack Success/Fail Notification Step #- run: ./gradlew test workflows: build_and_push_image: jobs: + - build: + context: + - slack - aws-ecr/build-and-push-image: aws-access-key-id: AWS_ACCESS_KEY_ID aws-secret-access-key: AWS_SECRET_ACCESS_KEY @@ -42,6 +45,3 @@ workflows: repo-scan-on-push: true role-arn: arn:aws:iam::419830066942:role/CustomAdmin tag: latest - - build: - context: - - slack diff --git a/Dockerfile b/Dockerfile index 4bad37e3..31018ee7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ FROM openjdk:17 EXPOSE 5000 +./gradlew build COPY build/libs/*.jar . CMD java -jar *.jar From a1ae168cd3a50f7f29123b7893449c33ec3d8d50 Mon Sep 17 00:00:00 2001 From: avikganguly01 Date: Mon, 29 May 2023 01:26:31 +0530 Subject: [PATCH 058/156] Upgrading Gradle wrapper, Spring to match Java version --- build.gradle | 67 +++--- gradle/wrapper/gradle-wrapper.jar | Bin 58694 -> 59821 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 257 ++++++++++++++--------- gradlew.bat | 22 +- 5 files changed, 195 insertions(+), 153 deletions(-) diff --git a/build.gradle b/build.gradle index cba60179..a8b8e68a 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ plugins { id 'java' id 'maven-publish' id 'eclipse' - id 'org.springframework.boot' version '2.1.9.RELEASE' + id 'org.springframework.boot' version '2.6.2' } repositories { @@ -27,21 +27,24 @@ repositories { dependencies { implementation 'com.google.code.gson:gson:2.8.9' implementation 'org.json:json:20210307' - implementation 'org.mifos:ph-ee-connector-common:1.0.0-SNAPSHOT' + implementation 'org.mifos:ph-ee-connector-common:1.4.1-SNAPSHOT' implementation 'org.apache.camel.springboot:camel-spring-boot-starter:3.4.0' implementation 'org.apache.camel:camel-undertow:3.4.0' - implementation 'org.springframework.boot:spring-boot-starter:2.5.2' - implementation 'org.springframework.boot:spring-boot-starter-web:2.5.2' + implementation 'org.springframework.boot:spring-boot-starter:2.6.2' + implementation 'org.springframework.boot:spring-boot-starter-web:2.6.2' implementation 'org.springframework:spring-web:5.3.19' implementation 'org.apache.camel:camel-http:3.4.0' - implementation 'com.amazonaws:aws-java-sdk:1.11.486' + implementation 'com.amazonaws:aws-java-sdk-s3:1.11.486' + //To be removed + implementation 'com.amazonaws:aws-java-sdk-dynamodb:1.11.486' + implementation 'com.azure:azure-storage-blob:12.12.0' implementation 'io.camunda:zeebe-client-java:1.1.0' - implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.12.0' + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.12.3' implementation 'org.springframework.kafka:spring-kafka:2.5.8.RELEASE' - implementation 'org.apache.camel:camel-jackson:3.4.0' + implementation 'org.apache.camel.springboot:camel-mail-starter:3.4.0' - compileOnly 'org.projectlombok:lombok:1.18.24' + implementation 'org.projectlombok:lombok:1.18.24' annotationProcessor 'org.projectlombok:lombok:1.18.24' testImplementation 'org.apache.camel:camel-test-spring-junit5:3.4.0' @@ -49,7 +52,7 @@ dependencies { testImplementation 'io.cucumber:cucumber-spring:7.8.1' testImplementation 'io.cucumber:cucumber-java:7.8.1' testImplementation 'org.apache.camel:camel-test:3.4.0' - testImplementation 'org.springframework.boot:spring-boot-starter-test:2.5.4' + testImplementation 'org.springframework.boot:spring-boot-starter-test:2.6.2' testImplementation "com.google.truth:truth:1.1.3" testImplementation 'com.google.code.gson:gson:2.9.0' } @@ -71,28 +74,30 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } -tasks.named('test') { - useJUnitPlatform() -} -configurations { - cucumberRuntime { - extendsFrom testImplementation - } -} +// tasks.named('test') { +// useJUnitPlatform() +// } -task cucumberCli() { - dependsOn assemble, testClasses - doLast { - javaexec { - main = "io.cucumber.core.cli.Main" - classpath = configurations.cucumberRuntime + sourceSets.main.output + sourceSets.test.output - args = [ - '--plugin', 'pretty', - '--plugin', 'html:target/cucumber-report.html', - '--glue', 'org.mifos.processor.cucumber', - 'src/test/java/resources'] - } - } -} + +// configurations { +// cucumberRuntime { +// extendsFrom testImplementation +// } +// } + +// task cucumberCli() { +// dependsOn assemble, testClasses +// doLast { +// javaexec { +// main = "io.cucumber.core.cli.Main" +// classpath = configurations.cucumberRuntime + sourceSets.main.output + sourceSets.test.output +// args = [ +// '--plugin', 'pretty', +// '--plugin', 'html:target/cucumber-report.html', +// '--glue', 'org.mifos.processor.cucumber', +// 'src/test/java/resources'] +// } +// } +// } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 490fda8577df6c95960ba7077c43220e5bb2c0d9..41d9927a4d4fb3f96a785543079b8df6723c946b 100644 GIT binary patch delta 20916 zcmV)ZK&!vT$^)&_1F(Jx4tca&z%&E^0N4ir06_qgan&7@z0@XurB&Zk8&wqkZZ?~R zWfM|JC=^;i+ZqBTRVxZG;qbh z6hp-EDiz@~bQV0nlBxK-?1;>&&zF~lpD8G3mZ70698uwcU^unc#xZZzbv$0a>U$Mm zRI3HM8VFZ~9$xmvitW{^+k1px5^li2@`QiZcJ1Ii!{yX}0R)~4vTv3lJnEH+YjeSN z#g*F9g79bff_OzQJTJl^< z8WeiLeln_d{Ci!w*aZ+-1w@Se)qUhx~3Di@YSZ$@6?~~`*n#tfz zc-zb5r^auW#ByMJF0p8*lP%R8>XK)|R9-4~Q{V;M*`gWWp}4hV7QGt9#zwCK5c)xeB}YnU}KCq3`dvF<^e z;fPvw=;$-kH8-%A#I{o+&Kr2o!uz;x;R9SZ@F7F?e^i}B@8C@>+!Q4`o&M)&Zw4(T zE7Y;?(?So`W_1}(*BeQpQG5AC@o`PKC6Vph<+!MJ%f`q)MXwn6wp*dbTj5SkOx>W$ zr}Ddh12;ENP-3rS#4$CxFHZYzYBP(zx2hacT5QKL$z@Y$OZ>L%p?qHUkhWw~**Xt6 zGxX4Hm;mvGh8Ec|bp5r+rlBz;`-a9a`MYqG>@oVcKUa+N#qdCsWo-#OJ0bmn0sM(!{H55Xi8G;Xvm3!nNGWzB=p%1P&HB~80h;U367g1Arsu0p`AMCB zaxQMByL;Es7&o28Aw8+z!%1QmPU@fGsN|f~H((O&_=$&T3Slmt)aexOqj^OC2GRKr zeI2pK4;1(i5wz+uUO)rce-I-_7o+B7Aq=qyhFKU{remBLO3E1`7bEf+5~?Fh@3qr- zSrsO?Eib1;F*TE-G$pD5+R!BfRukz|q8p;FYnoKa=tR7%_1@{e<0a61>yr%}+Ca+} z+og}lY|PxJ)Mx5w3B0Dzh`EY<9WSakFI_$;kV5lcP)h>@3IG5I2mk;8K>(hDt=pjk z004dmlOPT|e^uRcQ$-a2?M>)SZZ|Z5-~~JBt`t7MWCdhVESRdptJ(po1OIH z_1<`MW9dKSGdh~0%s4*j3=hmWfI2?-SNPx)!`a)klcb##JL&E@yT5bxoZmUS_tEp8 zeg<$FB?Ct>8l~a1fj+#e)H6}Mhxe8Afq@Tk*1#CXe|4NQ5LJLEK8oPHnlE(YA|`ZP zil7j|q>jr5dT_-+0v{`HpXj)%;~GPkziw5Nd4_?a>y>k5Pn2w#+werCBE8(CyW!Zb zD2)^FoaI>ld4|DMM`U`5AzW}vlA)()IdZ!8<(l*s#F|YjiK1%?dr5ef(i@9m6$0(e zdhWf=e~FsEE*;-8h3|R{{Ho(fuV9O6RT3uO>FDHM2NZXIS^BeNzTtYMrajAb7*3|r z#k=Bbk#nS<+aE+8Vio0gXImu^*x14;=jMISa>@jX5Xg5;*Jg+Y_{FNE_+#_FFu#}* zl^|CgpE6wN%wNY0M+17bdwDgCGMqR7&0x&Ce>KmPS1iSJY-cO0GT?X($2pFnUr9xd zDa`1Y<@gM99P?PvvB
pE_5EaN6aUwh`4YL;D+9>)q+>9Fi>>Kvad!&@A;afcz^ zZpd&r+r}#>p98AFA+V;RH91Zq$&mej{lIW^_k^~MfsRZvgi?z7ICayU{Hi9aWVEnZ ze^I2i>8a+XYB?m?f~4XkUk1l;Xm)Pq#xldHm-X8eB~iUNSEQxpsAz>yyjf@HRu^Tr ziFYxo5{e^bN1 zVr(B+ycIS*@*BDmN2VVlu{=Cd$17Xty-&fudOt`=1$&CVD8TO&;uHFSJf&~YGn~aA zSjC^XgTG+nZ&dJ4kW7XGUPUk7CaVykZ{Z!17}<;?MUo<<)<`u?|3{!bhpD4y`}ZGs z9K_*JgMmph2r0fYLt9|`^EbGSLufcv(lAwn1=0wzK^1a=Wgc%rc@I(L#>h^iaOp1| zq44@JeX;)qP)h>@3IG5I2mk;8K>&|%Ps}M2005{a001EXlX2A}f5lk|cw5z#KF3<# zv-BL>O5#LG2m-P=UP%ZE*kFhq2P^TCSayOFLU^&BZ6&fKq^B&DC6uOnSPHZ(rF5Z$ zrO?n4Il(}nEeU(+Y;^YNw4Et4%)WML>;Ct?Ct0$EN#~pIn|$$o_nv$1x#ynqpL6cD zU;oW>FA~uz{!uB_f6;$<=s)SdJm~$mXnv{C|9HqnA9(0-`jwY{O~3KdV>E$=Nn!Sq zmrFckutW5mUOLJy4Cw>NVz+1%(WnZSdT0)N#J~%|tf^ckxM!(cu5g8i7ILMBeLUMk zukjqw%oS`^qM7HR4xX>DU*QEFTEYuGbOA5&&|7?tm|iTJe{)q1c<2p2PvP^$_X5!@ z5zSK3Ec4Ktyxhapyh1cJ3fHP!=b_bH?;`Q9f}no`H+txGzEI_r9$v*4iAAdgWwpW= zE4;=-SJDS6uf>-y5zVD4H;J!9NNQI2GNI|^qG=J$6{5LPH0wmuDw;Nh*DKtv@&*rW z;bx)xRVsIQe<;F1l{c!qN##yK?Gk3~_Rw|Qqw;1C{gk(;+^cX%kor{aS9z;KCl%i2 z;j4K-r~cN-Ys+-6uXC2HdG!`d06EUl_M(eQF*Ue7*%*w z;TV$^jKz#(TU1Y_j1<$k>y05j9W~d-_s61fJ-j&?e;-a7sZ?hqWg1{)@_}^V;JUOq zV#LhIpl-%NEbs5$)ZMeKd)L^nH3`RF4fehRkFnHr&iq))tSYOg(09 z)uU->7H9Id_HWqG-n%Ol+}_T_K?hMaG0ocC+jCWKdnmN4tplO#a^Xb?b{VFu9}g;bC!(g`M%_aN}@N4@Zs0 z{YgEMFp`a(auS*<71%m>L8se|Z`Vg`(6L>Rr? zvQ^rQL>s&G1ei)gk>QwbrjtrWTbZ0~ z@vtFOjl_)Z^yqFQ*{AQ08p6@>K|Q)vPe#PpCOXZL2o846OtyiF^9JLweXv4knn{*d z=bd_TqItzMQ@H@cQ_6ycPzkf{&u#X2cX;e^81D-0NQo!x=Y+WR-p==9$OvS+j76t4)tJo5jj zz*$;JsJRFb9EuDh(x&D$8Ssv3!K8+5-H?K8KgA5QvA;KH%Zx8H|}mGNwndAe}Oj>xKPvQLvAILFBm=;fQ%!;YTKc|2YIv6wc0|h##g_tUM zuT@V)2B#unMN!93H)~s%i4a#J?thz|9Emk)xh7HqN zlwot0o_LA$so*{&y}5y3S#k1O9o zl5uSwqg$d1oq2gHgXySl8vQv-oL}5qg~yhbi^(diCLnZwJc-lz z3dG59V4721RDws)wi>apxTJWkfnCZgC#P~WcRXbk{nB_UpSUhLI3jKWe}jVvmnADN zypXA(XnM=lw&``MQd!QlB*bBp=rJQ}mw3pBBB(@Mksp z9DQD+BXmfk+vyN0+YAR z)Fc^poN3ePS52|c?F!$a@tu5^#$VvOHNFQ&rwkp_=!f)<#`p3Uk=C=hrB!A~Bb=Xs zkmB+)Z~l~!|0U7fr}6##fJVPhf1uGX=soyOC|ob`rtyRPWn3X=f6!gyhxjWRKP;NB z^4B!}I)6jsNBElxKdSLjeoW)X`L?XLL=x=Dg&Jl8Gb_J?+DwU`w< zMROd9C@`eMV&SHMe`(1S>ST~`@`G99F@duaqB$u}{ha8Hi{^PWSn&dEH$yT;cAM0% z7sUyCw3F`-B$(rQ{pfkeSMn) zGPMT^b9*3;J7_?U!OkgT5I2FqWcn7HWXop&ivm|kcL&12e|<8tJ1uw`0-K`-61!NI ziolA2bZ)~?1L9n47f(iRuq%n%QbMD5>Bkzs#4l_7J^nt7D-PWt0=J=6)(MS&z^?#! zli42;G$qk$e45YLfX-jLB)4LgmHqqTmZ$A~@@s&?uWI}nzYhEL_id=ZMB_L3%>usc z(%D4{15-dye-AVq7>zdUj>N(Z>vdB%abs??Zub&a2?#JP)mm`_5;ygo`A{oj*HmDC z#2lHyZ;@0W7B>T_bRrSQHsJth2a#X{8@G1B?mjCt8|=`u@=aACakE`k1is(W_-zsB zvQ9MYv#W+?kL+qtZVg8#DM~!mU==A3dIe-%){TWre^ZT1Q-SKGsb=|qMP66qAM!g$ zS4A>OaDDF5dCp8pawV>1C@p0LX&zzKQ*c&4ozIHmya!t9m!F0uMJoz~+ zmdBo-6FLi^(>Icg@3&r4bBo>^3b7|2hd3a@AYbYE)%iuX;1#FJH(|)Y#h|i)3#8@( zQ%N2*f08p$RRMd}aN<@vqNloz1EzcuY{y+WCI^N0j@*^X4P7@Gz_@DW!cGOvzPCFP z;(B~Z_4K9FDMjUP`agSIiR-FkSXg0M5p{}ge#OjJpP43?tdtv0B5BI+kvu8Gv~L=S zo4h+>0iTp1;h-l$v_M;t^~B_R%00R_97zHye=ajFF_`1YJ}Cq~roZ6)leB_|i0XAmoAl9Zp*W4IDl6jlzK?XdN@a)1$z|4 z-@H776uz*;a{^)k05kg_1gL|Mos`cWOCGo|>od7YU-N{53OUA(ttowMyZ)(BOz6YK?+^M?7U( zz0Ih9c25`X6uD+g+7MONi|&S`K0GSle@Th6!rpXB@PD^VXzd36^=N7e2(q@>ZYe91 zzwl(fu_UqCf_6c5@K0y6ADl3f0EjJa=|^}kyMZul_OX0rku^J}t;v@%?fE_b!{%U& zNNitxFZ_3D5o7F_89TWIEDY9q(3HQ(EOtWi%b-QoZ8PCcpkneh?jDLKM|E?Ge^x~f z|DlL(rt_$d3}U*0JmjZa=n(q15>W|#oEFe+^oiW~Fvg$E8-EJpPv?z4gYjnz#y;^> zMc*f0tMYrh{l$Y*e(`n&|J{=R9@5aim%fPhOQ7r^H#zYwuc_s_`V&-gf*fPyd`2?g zht`8H(M2?iR@43H)mZHG06i%Ge_B*7w`bWC3Y5s6P{Dza*|Z0uDX*{$`HN=9S_q>(n;aY~yET%Jf#vLWi8i0C|${n=r=){@Y zW02{TnpTnc`_)=f=@ME>m(nI`qHb!IS_iQEQd&k|p|7F`E$D0Xb$nIof1qzj^E@IY zl-OL~1eY~;59S`lTpKxmO+JN=D)iU{tstjDk6R;!yi_8)kqIm+oN!nZ`2S1HSw46| zdf-XRFgT41J|L>A8Ka7t+T&DNQ-7R%HGb!DnqA{}9Vf0Cr#S<)bz?O5I8}jdo^X#t z9{PMpC2D~QuAox75^h>Ye{-l6C~2eRw4Q3IT^gXp+9T(-x}4kUBu+|X?}&7p=m`fp zWlx~TCC#JR{CB}9F8eIs!p~Fmw2dYMYXr}VnrDf9^G}k$1g@);qXlwwn##v%;Xn;-wBs083L_xc}0fm?QetM znW4reS7kLluQsV-e{|s*Pw853>Dn^C%dci=<-nTKl80&M30jq*i|(hgaauj#yLgP& zoT~E9Ej_HS^;VV1Z&_8@99lbCde1|oVajv(o(E`wq#0|=;DoAL(5k#A=~6fLR~>Up zGt_i+;z3MR`kFIzncsDqRKF`jm)D=Bp85>69HovXMQk`ue^(-G*ZEpA)F#fi-q(&^ zhBo+>F}mthQ+aVAiV)&2KlKbOK8xO__kmMLG8eH3H zG3`V^?gHknqgA9Mn0Eu$g9z;~^#W}JG>n-MP$D!0UzxO*ZlWl%W*n$az-LLsSqeTi z>3g&v``<|Ke}d!B=w|u}5dL#y(f8>f{UhA~O>X2xbQ7OPH}f(Hs)H_&ao64_Q@ z=00F!6%yJQ)+yLIO(zKX3%guP&w-~Dn#5=vJrDHROV85_i2osI_ab^_IL8?}h2-YM zZWqwY&_@wS%3=^b8-wUs7%brn>HG8pazF>R*Sff#e_lZpxsV8#K;o-FqzH_6$UA|} z$`q=tLn$8V*n*2$Y?g=XA^!zP3gi< z_ZamYf9o8lEd##Z+A#{9lA#?y8QxrwZEgqU4y4dKrGrKGKOd)lnSLbs>*@F45G(Jv ziFF05OT_Oy?acEOoFGzUOAYqu>#je8vgzxug-#jT`T%J%AhsbO>U>vcqh|myQ&T@i z+pX9^I+)CWiv=K17~Ck%9_-+SR`URl06=#+f9mLD^=-)|u9jTlYN0;_KuWOs68a;V z#)aN6*00D5&(6u&b2XUWr5{`NOiA&l~=&zxpuy8GvOkjJ5gfRl>f2Se__Dldh&KwB@NyU<%A?H{~7L`*tvrSgV zn5>SlM3)@Kv%~JX({wKK=t3JosB&!;O&LnmTL?M^yj_1Rhau|@^gL443&7KhNU5iQ zjFu^ropuZT!_qDh6^ zlDIVQV&kHJl^d7PzsdIBQJut%&Pg8={(;arsC+j8@(c0385~|ZN)NkBe+#p%AIS;{ zP;$DNs|f%AEt7vLK7XQeqKP7(MLUTiOfgu(5FrU9d>}v)MBo66w2}_)ZpGdnB~IdW zr;!pbAG%` z!OK-tNyaL?LN2~i#j9{$!K+owamlYy@mjo2ioIJ7_sHS(D(=M_6ui-oH~H~q6#{Qj z@KzOdc$!;RD|$u74O0Q^6R~Fc%K~J@5cxHxS*mG56Jlk zReT5!s`xNcDt|^Xrs5(VQZTOIBPurGl8QTU*^eu7!$>FsXccci0hSNoB! zJw2UWy9EN_IGO7J;Hr{eA&ot2sbz72!zS*qw|WGpru{Sk#b4n*Dlo zzivh3JWcf3!;yr*vao3lH4a(&K-6eFZRsN;hSj*knI3RwyYxt$PWfW|6_^t`ROqs! zUAKlPuYcNOSwKxo-l%=z7P|OiK!uWZp zbo2~l?IU|-B2O$Ka{Wu%k!a&2UHa+D>ZP$AW`B%+mPE~=A!+OauPB6_MeeIeQ`*lM z{Yl#-{}M0$1G!uh_VzRVhU<=kuQ53Kz(76>L@MNMaBi4#XENfop*&iCT&vQSWh<); z6|niEOiyWKCN<)Q-PpUYlWT$~Ey^KwpN-r4nT%uP31(&7ZnHJ2ClYPO;C3?^A85BM z2Y-l}-L~F;VwXPRB)@{M3#<)I9o_;{w@z!*^9Va#QsAm#=CFou z;oAznqv5;w9(hg*6rW5+3|nB`|8f2_e19K5(C|b2NWqUa`~;6Nizbt^Lr)Adh#G#1 zpDFmchF{>98X|a_h7p|9a2!up@GA|!#%~1Xd$`pp>vA;8Q|>ODrpaq0puf#UaDso2 zZV$I45=M+!V@bZ>$>H|`%RQL4v1lc1P9$DTIM<;+;Ew`Yb{^3scjxNKx+lf{Nq@ng zHT(sC)$lhN+`r?hhJWBH>%{HM3u}o$SL;Ximn!GAS6Bor}26Gg(O zh+<8YaEb-jIYC5fGAH&W<8~xwWELr3_%%@~t?&)fculCHOkhi^9*>)LaKNCGF@`YM zFVEcw@d^%_Rxpi8kOeQ>pg)2V4`ga8(hrH8DrbWk!m5im1`Vd`Vm&^MfL4u6Zb>&V{UvlRC1X z(XxS)1vDBpPxr=8#CaIICK|<(Q2lheQ;C@ty3wu3 z?Ws)5$d!$AFfx?1n4C)sYFwc+viui2t$DiMc=pb$ZBB7up&l~q>~=A4qFoerQzhN( z1-4D2DrZSy1!=C&n+weHQX?-QNxMZ>j6f(~1TmkC@wwEB9k z!+AZXFc{t!wi`lIHTc8VRe$4EUMG5#=W0$dbrdLu;}d~6vDbEe=#vgcl&6|(Hs<$Y1J zjFG4=Zx2?2{F_44inW)_<zhgugT%UBcCzuEr~Cn0;!X$lsE5MYpjEm6wX{MGuKYHl!u#hOM95*%r>18pT{XL zq4ToA>KV4o)AAPSpd-WA-9KzNw|V1aQjZd#+1dPVcWbUcNCp;yn9ji6;(`7lk3C+I zo9(ezTH^MvI5SVSX@BRi(UhVg!|4LKE=_ot%}mjOLMn~6&Ge(p$y7`triwVbOFZz^~Jkzt5O^=iB3mTq&iF;k!cH}(D!lrp5Il{|M=wSVR+z9!fCx8z% z2xAunc5^lZWt?|q%{9*VWX&r$-;!7y) zD+-i#_xS=^cM3C8D4!%(NOGkl9~eg!cLth@Yl`LG>Kb1|3bUF@#xT1{sZmHR8OI#5 z%uQil3Nm&0h z7LQ`dIF?e@vJ{q2dcG3@j>`#jh`!dN94jyvE3pWxupXHC$-@lgIofm_Cn)VaZHpqth{PQ|0w%xt7)b{@ zLRm%pT})I$eJca?qi7hz3WDemR%8(-!6_8RWu0yBeP|y|Qm;&c3;y5I~jyevh z@*$X3yMG~_Uu6vBO84J|yetc6Wm%bn4k#j#5z&(7n=9?4m{t5+9cT=!N#UkZtj!m< zCrjMij5t10)8ZZ@fHU~p6HrrFm%{oKHl%R#ZTDejcpRJhR*d47D^$@GXij1CJ(wqn zfm?l-u%)l4p?eHlNAaX9gsSBtwh3Gar_E{`MSuGRM{)&IXd`aLb|TkGw4`mz7$VTJ z3Lc*Tk5V-gXdw?$h>3A4ZQQ|R=pb&=_6CNpg8Py<#aT6Xo@R;(u50`{%ddq@ICu4_ z{Hua*pMs5#qg25i!A^xrRrwZ8JKX1F15n!#C`)1c-I(QAEGIis=#XZQp;N+GlofP? zYBdKjpFy64v)0K(S7{OWtWo7dH?3w@mw^fHaiQ z1=^+-+s#21XrUw!1QLqT5NHB~qzFuQC!0xTXV%$S0`;KUT910yEA>9Hib}CrLea+i zKx@5k(N?WTJ?kI#Q$_v#W_FX^&5{!H`Q*KM@BQxI_xF2m&;94YhXAa>Kjxts77r;N zUdV&ZLz;gFM@9=W5)R7>xL$m?jWYim@HPzl z+o+qmvq|92$ZUkgK`6J%`+CVv&9u61uYkYJic{1VcIu|ql}Qe0_HK26^HAN%Y1ymV zIZ_CM$4;a@npqna@v1Sk|;}6k-F04D!ND3Hl)QyBz)49J-u_0 zq~&E7l9DjlNv5W7#2s`iLxd*t&?O^c`7tx?mk)HpHr z6V}q|pw?cBHN~+~o!Vi|AUu-uRi1P@H=>1he41TN19)6MA4fnhDiy65%i2~%}4HmTb(lc2x=s*YI3 zwl&jP)J3O1ru z!6qJ_gUt%IV5@>Qv@6(-EXgWVrQm-wb_pygh3M9dZp1ZP!354Ico1hPuT9p7g3s`J zJ3g!6AxtXx91ox8y)WR43ciFdEBFe&D&cDizK(MWzJYH_cv!)=@Q8wMV^YF*6nq!o z69_b=RNGPTef)ro$^(FO*RsMNO8Ak2A2Zia@KXstQ}A>ALcw|bQo*nAsDgjT@M{IX zVg0{Ff-uvr8JffFgg=V7WgjI$kK=a|ey`vU_@ja+@FxX-#$SlI3@`~%ZC1vJ*P517 zn=xABWLZM(P}sOd~^bT}F{F3ZkG$+f5;N=q-Ulu#S9 zHPvxzSD?9Ur!CtfXY?mugq!62gy!s0*=w&3BwBZ=Nv*UBr*8w6ZBiRq87b3tXKdwQ zPK+VP^0ox7jg)KqrTx27Pq*iR=9)`=E2yVEI~S6jTL#jWk#V$LsxyB?fl*^IEln5I zD!Qr?bH0L#-IzmX&Z%9^Y8z)$Tg=Jcy}nAyM`zNj$DgkdG7Oq_snLOFwcBEg!HM#R}(2b&o@M( zk%rWCmv)4Do8NS&ya6|{0W0TXK!5r6U9LNN{iN_cRPdym39o z6*RMg4gdEu+7(3(?I2ldf(op1x=GD7ihM_SS5B8U%3G+VUpg%y`w*cjqV`0K-g zaRlna!EprZPeXsYpC~KvJQ{<<57PL28f$2eB%l*rfcwRo=t03x0y-l-zJmy`o(lOa zLVfiQpdyQTXCe2UMy0n>vY21JfQBpYt-gu~R%UTE5oJ;99bUuY!~QdPmJeaSC+*tayu}k((c+#yTc9Nn4xGVt0%K_9 zuj>UpFGDTC5+xB*5)P7^o;5i|;ot=7dXN8iGAB8ZMST_xtdIWburYT~;i+AD9EVy0 zC8DDw(zAb%g;%fgmcHC_b&_^07=%Z7CQE?B|)~&>QOx%tANG1@kwPU5=12Z~==Y#3aO@M%%E2rhj4ozY!%WXvF3xL4#FW zC^QML>=1R!5LClitUp<{dKHf#=fRX18i!nZ9mDQC0>eE^!o&{xs;0eWcz@lr+XMN3 zP)h>@3IG5I2mk;8K>)1a;KzLe000>V005J-W*3vsKnIf#J|TYs)^GemsnQl?%ZDTc zQA5>WQuG4T1n;KHP!_f=*=^N-4iVacxHBKOR1qv(=&5+=Y7t5c4qg_ z-ygpKEMVC{9C;aL5)@`-cqr339R)eg8%SV5$0CDK_4tk}HX1d-Fk0H@@44kPte1`} zR_VencifgIHiduh^D3=K2P<4&uUcD8~wl^(8QFkjXECU@Q9p0E#8S{Gi?<-RX` zhPCWjb*|L#s#aja`rz@yL*ZE=CDTo#I$E)@Lq*0)PF=ii)hfb!%PX#O)v&o+=AI+- z@S~1kTPaN!XR>d5~)$Y?Liy*X7{{T=+0|W{H00;;G z000m{09MR|iM|y802eO+03ZO9an%)*&p;J_XSO$aJhngxBa5RF$Q}|lNhmgiAi)qQ z0n&i2yd*El=q%~X8%eaSOT}7i7i(z?)>@ZJ7h1vA1dIx5>(+hMwsyCzwzai;wP?Qo zxo>7NNd|%bK7V-cn|sea_bmT&?zuNlzy83(L{#INtkWJoPnreN6iKsCnnihNilr%k zk)NeHm2jEHU2G~$lxYFZZAzc7?5~-Ep(8&x%(yt4XZ%cg1!`!!fgqX?`xkrnRYmdD*4Uw0r9z zVx%K1oX^d*U~@JN7M*4FcZQRR(B;;;P}r*P?6eYzjb_wTvQ_2c(0I}XVCP#EiiYgf zOm8oK1HIl@lT(XbvYpA(5bLs-rZ$J7R!cI{Va2zam=q7hI?eEQGai!PsX>q38-nNN z=G@uwHKf-&Pt&U`Ox4_c1a}ATb*XqZ32)^xB8@eJ0l6ek;gw>ghhwOo>jX)=$xi-{e zC2a6=cMA*(GP^cK`;sic z{27h!(de+o2bgAmU2erg-2>Sc+!BicSr?7ddZN-f@sh&ng6!M`5el=xu+~a+#zTFM zk3C(XIJBMS@W*_(syU?0pgXMB$09J?tg!(z(3cLH;JAgW#)`eVPUFvxs+kI3C00V5 zyR3MkVy9Fbk*!4>r@;wkr=2vz3MzStSTf#eseqVd_4iqSown7bX8d6XlTbC78&BD) zdj%~Zow{=@sYc(&Cas~KsA(tT*nIIRculYdfReE6#(rSW76Wd!;*n4yArXKW9M27k zx&(`}Gvf}ZtgbDV+2thDgxwWO+8Mm6t=Glk5!1F6fhUaQ2R?lrNaR_@+#HWdT3(Y3 zg&j`Z$24PqB7w|UW5QumCTi<4{#PrSqS!K??1Ry>NhUi$YKTQ5W)$&ffG8GEMV-c< zgGob)wonwu7<6?Pm+XLAu|$h0z{~5h!j`1lcnbF)J!5CC zJmCi1HF_B?PDZZ?LF2P?##y!pyjIkX4`iI642-3JRmftEGNVplWDX~#8E{4>T}a%d z`EAM?mn11%LjtnGEWVA=WW>s3dAE|#Bh0bjigea2oG3^tz1HMrwW{o7qDy#wUgIwy z0VK?BD?RR*eqoN-%$a;WQAIy9l-L%X$fh6{8~AE!XRL1^E4MOLj`|`y=jJw{6C7}~ zTfyId9TunSrUJu)#fuj+_2fL9(SDFts?MZTc1Y1x3YkIA(6a`A{hj_{(9`sFgZ@Z=GUz+>U4uSEA2#R% z^g)9jp+^nc%LlRX8P(ac=!*v3LLW8gX2J#NO9tP|hYUW<_i6lP`SBHl@8>~-hjYYw=&q6st$s7EZB{+CusGfkO&V1t21;1l}L-D!foof z6vGC8i@$B~clf&oe~+JJdfQ2bNbQ|}z!tG!D5{t5ro;OAgc?%ghhTBuW<8^Kt2uBE|l6VB~Arzlz!Z3nn> z(nQRGV6S<(73{FAXwWtzeNwgrcZclW%A%;c&%`4^yVtbQSiwa(Z!3#}eeoE77TNQNzjZ#v{W7hy3}^5<-zWFECT!3@nk|g z;gpY+qF6AyBoCsZtPK_=ilXU+8kBlirkvr7FFR9}ZLRVO*aYiUc{*Xj;Ggk*sK`d~ zwIvy~LlG-eIZ4p{9CgfS_N|Y9$ISsr!3O_=UNrcZ{40ZhEnmNZ1L*~c3B4l}O~tV= zcpi1j$!fka=uY2byq&#~7>YFaie<|T{w-n+Tz+Tp@A(CTUlhR4%3X^x1#%qT&YZ!2 z;6DnZKS}e_7^O0`s=)-Qa&n?kt>` zV^St4ZHOhqT`F==m?H+$m3gplv@{QP$Ktg?$v^+p_+JM9TO6as5|vW;X#5|8Uty2N zuNwScK4I`{{5qz0I~A?LBQ7#r>~f(L%2~J>^s38kxIA#A%j@!`Ej*!GGhCXaJ~w*| zmtTB6cf2Pyjj26nXXoO7>?t_Gl?R`=bc6kjFCS~TXJ=V79JYGQaDBW7HDuJzSQSJn z*@RN5CMeZj5Hbc;jT{tnAZc=y;hH2Y`C^>Ll`>LmxC}lHyr2A-VOn&W6&J1)>1jzD zsG)1l{GlR^E~RjmhI8KFaUdI!*@bHsr>(v5*ly=)JY{U%g!{{Xgh(6%iL8np#dNNs zSKW@}N>bd^Bz-P}<$5tfvE5_YW3``FTr&R2X!bawtVfuc#W^eg;Hm7+bjg`Kk^N8a zP5I-*R(HjWdZUj|I_NS_PuKm=CE-}BowjnhYgM0_y1mxtyoV}W;2Z}SajCVd8X%m2 zYs@{&DNLZ?Bq*qVG>CRW!|Ib?-%k4UKXW^OsFvHon{tYE`Zq~WB6J0; z>xL&MSS;h`Tw%W^7PAooX5U7Xe66+_x5WcfrLb%^lTnl(@eL+M=I4H~8jX^SJ~gS^ z>p1e$*TrYO1C>u=+zUY3mh#BsyzMyIslFo-3ny($;{7C#1`;n;zmxIv=Ev7WJ2>j3KZh?Ds? zr>+WV0_WhB^Fp3Kx3y;e?v^!Cr;=22Y&WvCnaFzc%BIN~_WtzkP7+lXK|`+@Zw2rz zbGpERAG93ta> znp|3TKWU}q_mi)*;(nTPSnco$v}chI?P9EdtfZM#4H;@^J}sg1XesT(OsT_@_S3CI z>Puu|^(9id)R)NR#^@*MHjIcgM@T0P{pzwonz|9qw3gDZ&=h=3uk{2xgOp$Em2O6@ z595;qzG0f#UK;QPyyayTLsT$GvyM~Yp>cAqfR5*bb1h`8qdByS&ZX5#&h^f2897&f z)2HZmNaUrJ^l7>STy$u3C+zFS+ETiU?tx}=pyvVlEU3<t60F@C7^%-${$7(@4Nm z;9K(UBXn-97SIOiJkO_TQouWH@gOZ|3wULysO>r*9~k+TY`5?bgq#Y)TmS(tpnBE2 zmD#ofBt$mUN`hiHpwkZ*5760k1udd0shr-4byvfX*I?Dv)QEmPy@y)pI%)%dFgxga z>ZA8!#SNIb5dhr-w|_vHyc;IXr-k$-x);U?faf7P3{&>u`##WWw1bw>moY1@+naIC zUiu0QjTi%``)N?gemxBVGH&d9Asq!54|ZQo57039J%}|+NgDxMpGJE$D%9v9jUHx= zjuE{=^Vex~-2XaF$8U{-e1aB#YSi!wO&;UQN9fU1pp1a0SL82+2fZy7M`;m5p_%!` zo`#gi4Q4*&)o7BMse-sfzJa?xR%cB0yBoe1xi=0MX)cL^4r*NXa0+SV(zj zZ-(9}^8|YgqG0K+& zsY`u*TLtrjDs}RI!7$t1-=XiOG#o)Mk-K4-RwCAFB^lN2qgCiuwLt4772?8&mpdU? z;JuU1k?^Z=Lhnuc$S4nJM`^W-oRIr@`FNKOf&USh`6v{9fMx@KwAB!6B|QY0h7nGW zW?WiL(-rtNP%fqK(f6SMB+DqeA?0R2R*G|78J9XEPhvzAE+h8{QkRxhTz6Dc%hwH; zBE3lyLYFE5L25ucN=HG8NJpdzhykQ1ArP7r1D9f<2}l<~>54%FAs`Tn6eGR*r~)FL zFE8)C&!6|7+2@=+XU(0p?pk-9IkOpKi{D2~^)#0iA%T0Gz-ooZg2P zli{1^Q--B1{fL@oe2R?T@wYjk8C{onN>;-O@i2l^+iH3-2nOE(Bs1wn;!gAUt=*!Fn)aIp? zMs!`oF{h>x%pPrtjr8-SSbg=zqBYMvR1j_7RSrnkwX!Y} zDy6dT`?T3nifn5Rbx`STLKw2IbuE5@1nbPDAP^cMQ5M5 zW0Gg!>G@bY~Duc!*e zS64OLMGA$YU9P+u7Sq_rn4J}!AlufsXzY(R&A&LsBNP^wT5DTc#xKp}p;L3irQif& zQu(v|Rm90CD&>)D2bqNM=A1C*o^f&)uQFRc<{YiAVQy^Hk?r=#3^P^{EUyz-?DT(f z`9!ZWlmSVz7bvOSYJ;lqCwp6xGG3GOpeE{Qs0S|pRQKw(!S>N@e7ah6^If%L{;sfeyE6Sw zC<{S9b8(8zm>^h4Eyv%n!+zkgbKZU|=fqUZI=xF)Np~G?6)GOpCXrefBJHX6Xkl}# zi$3NfN~Fagewl;SJ}1tCU`N9nUD6bQnP|@CNMCSN(6?UisM{MUc@#Nt=*@ySz=`)D zkxc44DGMr^e7`*COXME+krOE8qY|nPw3Yo?V4p@y*uGph-|e(t2*1#3&R1b^dB&#- zg?8%?;2NGZr;_UNaa?OmFG{m6iTv3_E-WLsyCB@Z!_QYAa%Qg~wFtqk_)4p>7R|9S zP?7&ae0EwWZ!wOwgqkZz~m25LWi>nc_faXDMQrw>Fac7!I` zh!nrDvW<~z*9cYED1{lOQESCgYy{}yA?BxUa+_r~(MWBDTzcrn9{Oafiwu1B-1W+< zkIV_~ceRq+-w)nUOQ5qz@U}QJQtXBQR-R3<+`Tk|2q4QHgLbxeAU5dtvf2^maZaQ3 zQAWZ0&USX#Z?TBGccr6_>BTgZRRW*z4+s7wr=clx*|Lc1qxx2GxL0nqX~6u%LQ=Lk z;imB1ZN>#TD&oz2R#ySFUY1aG&SI>pWXd{bcK2BThw;nm5Gx#;g)X!%I~aB zkCQ=l8#848a4`1V?v-*b=1Dd!HZ5^U@yPc+gcsx7V3Wz0KJzOS{D%+?!>njugdsyHtCIJm%(?nnKA;sr=_32j=Nl}?45e>^&Q(KGN@^JYDjA*XG1#g> zJ7yv^#klUw0$sC7v&UZIhq0vcNI~n=6y@8F>Cs5|{5(c3T%vMgBk^3RztZ=!fnzRQ zaU1G-B`! z;U;GtEf%uwJMSI5y#7~#H9oGz;F}eDKUsKRszT2e=0ioSdIR6U9sAF+6(^2-G_sP~ zd|x7w1$T9>u~NOwbM&8nAGp7$FeR ze=74DwooKYxOei!6l>ZyD5z~ z9qho|@K?rKk%;+2y8G*sL&SdH3$8fDyC@mIwJ^lw#l@U^i@gWGiuuForPzY+Wert?ugmTC@5>83I-^rxJ4t>jbhyQ&lvq~Gcg=`4dZ~TZJd{eV#JjW@&RYB*a*J8Z# z26^g0Wb?K;TfR@4s2d_(Y5-rGtf=ML)Yx3{xXA9>r!V;~w!prq4>d*4haQ=ux$I8N-S(*Cl zu?p|dxzqMEa(9O)gUk&GUZr_ltl_=t*cKUsCuwNMAGg`C85=xx;R*LBp(mT^eMni; z_}y&uhJRSk)NYFj;#kVzO_Et1vb$H|`FuTKAx(lQW`Gum!Q$Q5Ejg8B zjV3?O-5H#~!tnVA3|Dy&Dxs${Kk}N$bBU=KH5s?*S~y(gpto#6EleVGn2?^!Su@AE z;d46=6Ss^$pMLK8wixTJL+5XOC+w+)Vw*k8!a9NjtUbR@$10D#*L|Oh#W&#Q<%W@di^6+DW*+Zy zZyC+a7Pz~=qY}(%h|dV#6urN$`pis~a4650_d~#zMemdrGRZ1`O;Ii4LWHYq;*<^j zkaiHaE8pj0YUcBIgmX$-9d!hF%!Zk>l9|N3n9cT;m~CgQUn-!L@O9UaQMzw-B;i8Y z^4q(fCHdts1XC)-2f~E=ziw{@R@`*8xI;c}bP<)~#1J;J-1t*Se^y|gU3ge$)GR+a z1?{a5S21nMT2`58Q5F?1uxiRt*B7eCOY^wnHv1>!_HqTXKXt0=&{BveqY*bLap19d z4u@!u&EN-RmhH*b_mVrvkvj(g!kn4MRYq7XC1bNGY2e}lKH;tNU)xl?l98^mSBXWx zB4{Y=uy2F<-Q|PeMg77>j?gz0!5smq_(Vhqm+*UR+rWN zh4}Lcn}QEVFzaO8eRzBIBx%D35g4Y^id%1=ou$rax}a>f>vOgrIU;Km#sprB4H0{m z0zY7pQeNT2F=}_lqX%1&u8)4YeZ`54WXFBG-rj$3Frg(-owM=NQnr(OuP$v_1>O)# z?MAy=gS}cg9$4dh|D)DvoGT3{HSXgP)7S5HZPsK=zi7VVWWx-KZ4_~r5!(S8D6tXn z=-9}TUIG8>>Zd#?k~}R}_e{`LQwh#@=ATL^m zbyS$*Se&=tHNYto+InD@1)sL)9AS!64yYMCh_uL}eLs^%BX&e3(KG9D#Q!GVS^N&T zHNgGGWk3j^B#Fp@@k3tF-)c_~h&)mM>A$BCMb1bPmy`M;0sT`>=06GqQi$4NA(Bi0 zhqEN^_!mvURD^Uc-YXyg;=VD1H;1_aODU3TW0(qjHhdQP|JQ~%AP~vl$%xs`$oxyx z0_aspP=ACU`nxq10ulOyzw;02?*@_F5iaQec8J!1$9;?&5OXKFnw*Jpd&C_pzQs)c-;6?((rs{>*i|d%M;xHwXHQ1()Ce;e z^pzV(3I^#ABH;KOlv(0GK5;BWtp}nI@HcMYc?2l%0t(*vrUfWPl3-R46Dc}IjAllWT(z@M5H`*YjK>pAVK5wh3hW$Y z29=}#ccGmRNkPwOZt&ZEgZ<+x%*No-d7`va1P N7!qZdOh*2;{tw&n@7n+X delta 19585 zcmY(KV|Shnw?)&iv8|@DZQHi3yRm&Wjcwbu+1R#i+eyze&bRkJtUbpXd(T}p4>2?b zQR@eb=w)>9$_)tywgLkNCY*qZMwCznO9gDI;;Uo)+qB_;ABR?vqDpN3-OLUtQ?Aii zSY_*PWkr>wgi^(ma=>BJaj|Cs@rnPoqiXn;uc%GFJ-K7I!4{QWF9a4W;F;~)|)OwB?Omf*@|z6Y=w z*b0GL79LfGIpctOm?RL)5IZe?hA)#^TI32)IN>Dib^0?yU?W7biV@W4VF`#zZrzWc z)=$euOEJFs7z@9gcCuLDoXPZ|niG8wE?6fdcuLkA-Y<&i)_f8#if^IycSt(lf-cFKNu7vuDa8iciS9NKE?l}=5=0rAs3h1OHv z;#YEx+sV`H=3J=_vrQ~`O4O4Md?*2H@P~v z>HButi8wm(uN6LVCHkm^jPeI81qCdW0Vy&pV50l`7Qt4t&eT@1*mm5h1;AJ|OX_*A z`2ZCOMorXhkj@b7=%T(o6l27nl*7c_Y`uOEy@qcNo#K-mPfhQ-0S~8uQx_E$__}kL zIKFzbmaCOawu7o;*(Rbtwu5Dsg-ya<`xu>v`rYVSytxa%!e7Y7T4HSaMw6N#jz-;; z5F~iXS;fjX#7<*la47f?M_?fT`A8fh0gJ(Stf#TDQ`&qpn4@2y$hGw0G^`wd<=&*9 z3e^&!(|p%fulWXv0pBBqweQTSS0;=jj7zPCt z&RBJR{PFlpvyngYV8fiLUUFz;MpU5r-DjPCeuv(vgl4El={TXS3W)v%GeX_1&$O3H zu!dS1Rov`)p~C7iYl{PO8x(B8&Fu2Y zKAxAyPIer;#I*Xn4>lU`z$WVzwd4VUqFd~UR4TW})sfMac!|C{BU^>bI!HNq2Yd7g zmGBXlGDIgjDfyU<0z}W+qiFSt-O#wjywfM~wuVDzuyHQR|Gttu6$%7Bp;(kxuy=xI z9Zc$m#wF?5-AMznt?qm{4Y`5th<>9PvQYHcgKZKKa2pY+FY`R8DC zU^r4VBDw{5AW09}q41j_mg(2hLXzfV}>K5{Vb7083Mg@&<3HN9Or7{2BB& zdIA!Q-M}wYB&jJ_UrdVGXu|G@V3cB2*sO97eEEPk?9i#kb6TrTK++!>MYtk)h_#gT zl@;?58#GbF6}&z2MVa+Ni2LD>`cb69t#E)B=97-LIBGICdaQ0*kAG^et|E77rjjmG zj5A2&sCC&{KNy8t(oNIX4uL&n0?dxAeU_1N13EvHQ#FQLV!aMSy1gc29pF0do@a<~uEo-!0DT5u zUn1y)^3xXT7W#tzEelC@@zM`G%{a5#?cluFq~9&;f0&9uWcUW6cV0g_s}HJ@W4AI& z@u5*`Y78)_SSL6F*C@O=;~(QPiGvVm?egBzfL#NsL{1_#j@ww{)hsQKg31mK07(`h zA_WY$GOLvIm&y2Ym!*QL_+`HYG;5(_7-eu;u}YPR?DKmaePNg+NBGSN+CK_~HiVhc z_%Otb;&(A@PtYE&=Z%z{;hcFovC|vNGSQMjB90>{p583jH^}p7aI3L$bmb?V%rBZT2AHxYH?qC)y3p4io z)xeAAMyM?Qg4efY2o*m zMeNb=9E)*%0oVoKZeMApx^sgKHT&&xUgQORUg#h1#0MYODg1_Gwg<0qU*d_e{I+8R zdx+m|3BIHd4K?_H%HW!pIw-(orvmzGAtvdCRZwGR`tt?sON9Pk5#HC>z`0*1GEfy< zgZqO2_@OJf74!Fg|N0gg^fF8XdZ$GHcQL$t%Lx4iEDrpAt%Ln{74)WxthSH;f-IOl z45Ys>3Vz{ltD$ulQEju_2#!VDSL`CP`$!@>6F`Dc*wt`UtV4$}YKRH<3d~JXP2QNP z)Qex_%Qyz^3ea;ni-Hx))ahJ78q>L>EP_a~J&A{Yoj+;iw5_EAepa!VH&F{iTyvO3 z87_?h^2HmKh1IMFSfV1u8=x^SRgP4r3~KzOPWiFyb$M$x^9E}9Vz!Jq6PXHXTaal7 zi93$SdX)p;71{H^h1!@jtljE)S=4p;SS`*O=hBjAt?9Df>MyC*mJl0Yv*fbD&3hs`rbTkYj7^|hRjGjzRk6)& zgRs%mdN{G_&)Jl}?|b{3qIlnM2P9fGBpylW<@xpI_BLi%=9=^GdRlca$ZjnYA3fZ< zX5Kggdt$}40_GB59&1H;?$IZ^E-7~CZSM=|khEQ!8R_Euu-05ZP#kO3;X!BGb>aok z!HE~s-T|2vC974t@;I0{82~<=JwO#zc^*BKmHlu~o4^M`9wxx-90(=EZ%j~r%qI2P z2%}9~g*F_GP^4R_C-xB-ET&I2yxcQ4}0>RO)X%@c{%|~S)yE($Hv}pT~+MZvgg%x+f$%sTJz@KrHY9rF{O2Idn!qwBlYkzX5_jDC%~-gI~gLkjW7Vxxv@Bu@icO&#mJTeBLd_2zT9&jIcr9G8ZN6@sL&H ztT|dw&orA`Ja0aXtP3_VbpDY}YgS9+kKrtI)=O#M{n>>q5?9J3BPL+gEk*-hryi!p z?C9M6CE^s(}=faOcK?ARm^O?OF#j|n<7-Q-aXzc7B*+R z%SY67P@fL z$4mb~AYdk)azP4rAnU%+BCnp^)=PhhH30OuEfO1+HBeqfu7FzVQQ z2Oj=_C8LEh5rH7TUZylV#9@BnD5{`!mz&vmK*CM_n*>?cWE^mm4OV}s+KX>8gIuP^ zRSzhbz*lW>WS4Wu<;ovjBmWV?&u3WoIWTilsM^@-lQWqc9lRlCHKtIU89OIkRzlwy zuI;axBr+DyDpL8^!^Ry@Y_$Zeehl^@6LNVqG8QV+sll1%4QNy1*6+LUA4c9dz)zAU z%_WAsp>PI0odY(3655fYCk{HdjX`fFNX&A!vf`_|c`m}ng;Dx>3*_;$lFqqpAY(sBD9@n-(giYE^eCZ>2iSw!mVZoUV2c3LlILHK>%Yau4dCcg#?8}{QN=mP^u$1z2 zQ&vV~6QbmPfvvagvfH#GHy;^}udf`A%lvnbZbu-bV60U>D_Ld_j(PMbw*vBUb*b&MhPH1sjLcYz=+#0*69?+5kpB%#8D)~vC9Uy1nwRg+u% zfKBQ7z2xmj&?2X&1zWn@=`Ys0)pH7$54g;G_XYE)2*$DAp*w7eCJkBl{lPHZ1|f^l zPgbFtw=BuIPW@?^vnQ68+PiJYriaEtA`Q$o9UtAvt(DUBSEu_Fi#MVpa_+L4m8~K4 zEa0rEyD;!U1<_Gkob{i9SJSA6V=6N`g!d#WCAH`9$+A`dgJh@rs~n~EhdC- zBsd}dKTJ{&_>gkJVJmwvh^RX$U$bev+)zF7 zI&W^Y7Z>R0t5{;9O+%K?ph;Fj{*5`1AL57$2`8h_5R7L|)kC)UeU?VVW<6HvzA(c{BV=xs~G#|F&;V*?C0rxL?`K4nGAxZgUdZX&rkSf6sp{nuCNwN zXf<{epe^w^_84sEn=23g$Y;QD*v?&PoUArBOHU@c5~;k-CN)%R>cUrke*tRp-k?F2 z#)zW`pz1u0Rh2T^vc>mn3%aXbr;CO5s*CQ-tB7>X&dPvQnAcjc?kOu^j4;Cfv+fyG^zn*be6Lci~V6Ip_NZ6S%KnTBJu6_p{1|Mq6b zr>l|d@7W<)w6=CL2E+5TD*=N1y;`gA8YR>}u&yJ~u3p*Ptgjy8Z4vSOZTNCdKkM;JC%Z$P< z|M{PxVIgx2RvDnzXS|$i6SS5+T#iYyalwg!FRrc0JCD?k?S6rpe16NeukY)ueN+En zKgi0hXEFT38}Pgw2?e7p)hog|4Mj2!WkMTlVAZ9_8TKuesDbj>dfra67wygzW#z#S z?U!S!)}bCA2wem|%N!F~q%4!OY4Mm; z?Km$x1n(7ovoc@ExLgynS@lj4Zv$$7-$z_O_7U>?f|j&xa6w~66%MTlg6l8N*g*5W zKhMuHzCR@9aK>E|-F0Jm{*BwC8rqG0z8K-^FM{my3q<8@a?h2v3(};KqLDqc zzP#9(ua8A-jyx~2s`iM_pLPbOLdCiGlDcps?0l$aR3PQZ*hll5fg9$IIrgs?ZA>}v zO-9g2XDLb7-g>D8jV8ukTbSHO{q&HkiQ3uz7D0G!*7RbV^5nHnFESkmw>CiTX&mGxk6-bR=*>6qb7uN2#|TER z$2oH9p2q;01-xSP)=@7nkRyLseTuKJa!B?Cl9j6$$I@EyR_TSzqg8oM!Hi3l-Kh-a zT$ZQ0(}k&?_2L<5!A;D7mqn!h6% zL1Ql4`$pfM?%AI=m@@~{E#CR+lBZHmVxKF%&O0)=?7(k%tXV3Cc{JKXt+PVtp{4ta z7Wf6xPr%FPkNS86EwR{-?QkZaWwi8{lI38xA0O8wO)S_)cq%fNk;RNdGWg7wo*;5* zggsJU+riNz&do0q%V2l%n8se2u?}9qe?5)(aNWOBn0wm?;lk3zf{tRZVXXyhI>2zQ z8_HAXqrRIfa7FxYUs{S&^S&ncecXNU65a>*9*+lpg&BY8zdZJ~=IE80uMQ8wsh)p- z8w|)s;b7BDIXFvT!N3p`m>30spLyt)RsyC%l*)nB&=D3Su+k1cgus6~{61`yf^ly# zv5$%8;#VLaEL;nlN9QF}QK-lK(Z$vEeD<&d{|tNp{#V^ z{ov!1M&LJc`G}F$;CCVw;Fd`O(?_b;9P`{!d{DpjY&hgFIOgS68HzX|hd3o7N9k7D zryMS4$P$C^C$}IiiS!qN0AmPa>C4aV!oBgA@IN)>E73Ku6K5;yb6D1b^))5F*Kt!< zS(lRS?$2(n*J$6x6i!MWi(9h^tj_AHw496rdKNTT)9N1!y9{GW07QSK(WmC8t>kJc zZ(XDkhVgn!Lrrh{Xr7wiI`w5HrgXRsobZW3_)qXJI+K1f6=4JML)9*Osm70I!+XJM zbCGAapY50l_jI$C$U`dNvOkO2XV{ERKOGBBBa4s`aWzy!SowFR+|KIAqVC9FXqi}j zN*Bx|+wBlhYNkxJ0insC_RIACpcGOI4%>)9#A*#>Z3^tm)VO$zZ-XrI9rO}g;Ut?f zxF1@nHI{5dzd-Ehu>7lxQJa@HD5WeQ3Pp4&J1yPLykosF4zv#E{5vq6;bv4(=8y}V z{bQ23xcQuxn2UK8@miO-UxsVZgBe4oa|+nMh?6ea&>I_m0(M9UQs&2A;v{=?PKNTy zbAC&q#kL6_5uGo{7*5hHEjdwm)!cC1Q#&W0pa1PcTdDPMV*An}tIlMck_?mvwv-5_ z)+$9_S&D;XS3VpqiWcZoP^J6r)>>L3e*JMe!bb6)BDejtnIgIiyhF21S;bpcE#ugK z;b6_Wuy3Io2ENMpn~C__REeFO40<^sHG(LkL1E>(6iufRBx>FhjhO5k582vI-5OH< zauCR&9QSf8c4wb>Px+Gl=y6L%q(!?_C(AQpB_jQtxAbp+UzAuvD4cI_;p!EfSCdO% zTYX}+b9dKYqW@fjUWmFCvsJ+lOXisSv9vw`0$xNxpa+@#{8o z2{f)4qG10t=7&f><1H;BGoN@rih?zle+kV?Q}Fgq3%E1i#990y7t!&q8}^jaVbWaQ zU)qNqoPTzkL^jezk&q2T@jic3`iUXW_)}ZzHHvH$3jd}$R0{(GIgpJXeEHH>Q<(dQ z3VmPNK(8U&2PQCoH0Y_OET~)jZ zL$e^%I=vM0oA`pY3C~<58>hfddaD1g@YJ#}Z4(Yw6-Z2GNpxp&$LtAY5y%>Es-lTk zy0CqgJH3KKUWKJej}5*CMo*T}{uTZuVyqy1iupvE3Az)W=_OBy&?~*NjL;BjmeLYH zr_$f0PihUlEV)1F^UlNOhb=QVcFIDlim4eRl{?R!S=Fsgn&*1vUXKXB(aP(k$&*$G zm2JLE^Dw+kmYDjKs4>*)j1n7m>J_=f7Vn%}L?w>D12d#=t;61xEWXDNBn)?v7L-bA z>%0WeaS$~2W9pod$M!rZ!aTw4CAbGLfO4&*^{mE9RFwQ}!lg0uAZlQ}Pu*9wn*Gaw zd^yfJ;#j?S1uBXtcF?vw0T&}hvw^4hXu!RAZoo-Jr^SO{Ggb5GNUpu^fuWO})@-$C zoHyA5SJY8|3%e5Ibvc9*Q=f6-ZPd0%6Esw&P~|FLGRrZ}b}pWA+6{&kO`QXnNL8i! zR+pt(_G_JXGlZNIy&5=Pi~F1<=0?|`**`rlX}yMGjOvp+W6QUANOE9c-G7HqwdE1- z3RN}WwLIpnaGznB9`P>NC&cZ}KFQtHTaQw;=2w83965G>UkwfpFx#0`JVDsFB-#=@ zs2iqrV&!|1k>7|;GCLX_x*z~J%2hawxLp6v6f(o#r@<0Z6{%RSPjq)@$|V)lt*MhF;RPPvwmC+r_Y{0Xh@;BDP4~{F z!8L&Lq8lbaK!(j3;a#eRX{{-#gr&^`zO8_C*dd`j%ZX65iv3`|BZ%^l zWGG>7sGVD0GL1ds#gdvX{K(ZN5nm#G=dZlrWSQH>r^WER1mzorD`#Jg(;I0)UK*>z z9G+&DmehNJ@JG0M!ukn-`5c#I(o{HN*JQx&{jgv!<%l*BHmr5R_+z?LEM$el8DnB> zI3pOT8(e3F5dqss7VzfS&_dc%mu-MFKLf31N^N0`dt}PJWCX=i7r)h+6s-}kNy z%rZ%g-IYB}tBvJ=$wzYAKL#xF<>=0Xa;Fb)krnRHWV&rS$z6oxdwcO*>ROfn-H^I+ zoN8WVI}N1hnpwJy0++0X4$fSQTAl!Q+{($FW!AflJ&^wbZ z8bpRqN?lt`Wt}CYZm{F9I#1~#i4A5e&S?1}N90okk`p43U`fa^t4oLgq!p~ALBo8M z8;?Pj8WSqc@@3~T^|ytsxhhR7Kx?0Ejr1%!Y+GmURz~wq?-4Xz7phy{lKD-VmbXMJ<&w(=+5BaG=r%F%^YI#l0O-)or9^$AATuTlUHhdK@~kU%GY{EmuW;JTV3)G zIRSzD1&Z4SV^1RimWKh*^7O%Q$NcwIKTMbKZB9u9+bToH#+h-KCa_;bgxMDD2T34g zVb%pw21x_!`30T7my&IjWY>9;SB{o=(Ta`8?9jW=uP*ypjVek0bK)meabJ52&ecsi zYUZ5juPOJ>t@rNdcloK$m3@Lo#t?LN@f)47_!$`(oY9d0Rv|e-9BN-^U;&F+ zp{G-%WJ}1+&nv?TTE^X@q78!MBP3kdPB{rqgQ>}^dk1WT-hvMEPP7`6c+GJS#4Fg{ zCb_qE6ocvdtsnbQ9_TflZ3NGI^l~p6+1*tH%AH*V%$)%lvnW^lhwft@*jx0bu~kw# z{2QH`eeWV3z;|Nc`)&YVm0_SNkMQ!eWt(RETiALZtKGawmMR7?c$0n9mV#ey5(b_D1!!k#4s@!DQgIf#w zyty1!(}buOpKXO4rP8|b>aEmJDD$yv!p<*xR^D{(H3#c2 zqnMVLek+BHn(Aosok!FQG$`YLSk%~|(aZO0pv&OJFF6N3gdSq7ItFMJ*a&Bm&)C8|42-v~}gQ_y? z?skHBi}9;CK>=J7V8Tz?QrMj-+2<@i$=Fyyjjqeb)Zt z&QmU=?anu~SmvAeUJtEhJk}Wtwpr4#mSENiXjm_E@N)AwuGH{wu+eJPke zP=-l2>`;co+)&#+oaB7@JGvffy)DetyS#AS5!bMg>G9>4R4)R4LK25yLhTjWyU64p zQoS*0=aesHfjQJK+YZ(u%@dlOy~{s`_fis^x;29|2j+76VOXMXLtyD!;HC+O;E)EVG$Z4$^ha!8D8vaP^yvfSV>dgQR zMG<3e#ym>1X$DsAY&3ewS@-e3l(SQk4fLq)lH!Ka(ahb4RIrW)aGdDurFYiSRm}gk`lXzgH51I} zIhWZ%JtkTni_|-x8;!BdMK{a329E&DAhzZ`zYPeU z`<0D*OMOC@C%MQo!Za=9BHNT&vY=w4WYz5psxCs|Xy%oTJU zq7rWjz3b^nU8wXsfSG{`W5S$#L2+9XZCkyoS)s*IeitII`NcMocPZV{)e1Ppao4(S zge>AfNcBv; z{jBv+PbV|(6I$)<))0|uDL^FhrnXyAKDs@PDV84>yr7oimI%t&cIn9c###F<%9+_H z*X5N#ov_O*{vXkp_uORQvKP>7{!G3SAM+wh?a2*=uhl#B?;ZCtJRia}o{n7sRzpC) z@aRmuo3xX*L>7OSTx~u@0e(aPl9uL$8dB*m`TBgo$mOa0!P5hsleZV=EN6a1#lNYU`c)#C_%e8g3Y&QHUFWc0$1pm6ZC?1#jP`6;HqST?UE@`;HX80Kf4L)myI zAoYcQ^Ug89A?RsW< z+MU^zv)b`rh;qPU;5B<-Nz+Ll^1Ck^K3V55>^5oO-G1OTFYG2I>~;z4W(-<4G3f-q zbg3WSr3tJZ);Zu9u@a0_N2tH|!z}{xDZbdwxIl6vhJTO1Ymjt>a;m_P&;Q#k##02a z_q+ACuD%@cAGx;_W6md(KZ+^}kMVATqt(L(cu>-Feh+cB-y~eI-r4sYALgrjLR(Xu zehJaQL~3? zrpQDdyhf5@Fl^jH9K3r)Zj_%Ok<{4yZOf3e>PhH}_m1Ba{4%omOyt?dOPH#TTi4>x@|b?0_uehw>0>gRHI$ zBAfj5#A?9NNONe|-%C_lb*(;9Gq z{unp<3~W1(TWc{AXz~@q!)!RNXRlxZ-P8Y_7|<&nJi9=bjeI2rMe1Iy;k`cU&$=6~ zEl5GWm2>Sk&3aeweU>W|25QG+h&3Q6ib`Z6dyXK>pu`{pyv5mOymvLT%8P};b2nQ> z5Bt~b3O#n4D~;CU5x&mSi`6+auEwaHv~zY zp!m=33+662O!SInsD<-2jtVcm9kKnr#Y$qYIwgF?;7?H!Z}V0&aRIE3-|>((2;WWw z@ho%MVb=)oT)rU5r`J?`U0KZMR;(NObw}zqA)0@ZP}uDyhbil^8viu!6Pg>qxH{9a z)N0QsMMrO??-vbrSzgcRA~kLSa47!8C}*hkB6KOkdfomUk%D0xm*QokprUloMVDFS zVN3ruq&VQN{e}N9EyJ$B%y3-`t5?#7TiJ&=$Yhof@ZFQ+H4%ixD~3+DL-F<6eNq>9#XCa+SGy^2Ywzg!2eo1;f#0K zrS?VJxINRPIFnE5LbA5+S~vWQ26DRqK!`hFO(yDPQ%CgB7>S6xT#+`M-sjK6u^_Hb6$Yg^g^@G+}65J#Kg$xIr zoDOl}O!h85asq9;lNnaOu?n-Ps!9j+Uj^1b`BPO&W<|5MrlXqzvi()nv5DQ`yXNOt zB>(TYJ8hKA^ZU#DOVImP$Mb*M&z<0Lw)b(!zqV@}RQlb&FxHI6vZ_BOap&aXiX8~j zfwDP_(c{J+NlsGxr$l=>d^m+areo{U<|g3oYy?!(9b%7fHhvK9G!&+lI6B4_-}vMi z<~mCMmJM*hPO;V_Dm>LgE$nDh@xmI@P3okd^KgvqBY?X@Gr%k4F`&p9FVZHN9J_Ui zoJP8#iK;O7aE;v>|5bRO309MI+$QV?11v+fYDk=ky}62L5f}b6Al|t@XcOKK{!w)h zg_lEIsIG;S6*XrQUo8D%lU$|?{-JQXTJG2|v<}|M-@{vIY8eZ9)t_sGJoVfvy0wei zrmw+XAsC%qmjao#@B z%zwser{k(jn#PtOm*@I;{;QeO4(M>u7rk$;LTp-ZVYjhscaWWIMwEk!!p2X*Q&%S4 z(q6RDjq^AL6~ns1?qavN-p(H>O&!QB+QqD~wykq^t8M&u-f4s7HZN zSRpRMvOU<08(AmaNNu!Ih*H$s5j!fNe-7ys^m$gV2hz8(rLt{a*hUn`197sGk8!2i z^Gg;K2^o>Cd1e((`$|9`tkJO{dYI%zliW46Cw$kIPC`P2etW>EZi*FW8U?$=o`ovS z5=dtq0U;a)%~rxC#8~`*ck{HVom+GSW`)?kH>mv3J< zZgbW5PGvH|n^QaSpVQ1oShOV8Z5)_kJejp&bGW9kW&;GMNEr>v9X#g%djUx_d6gPx zWVS3z?>7~F;3pRAUZ^0U7%VNJE1n^IftJ$B$3d>at4UAs^ zMMdY9_#9ob#b_7A6PPtu*4| z6-l&w>i*!JMU)HU>Xl|Nd+PmAtDrd}>GFo+t5xEMH*<>DjRc>gu}G6py?jd3u2Vv5 zoioIF@;KK?6Mp}g4O^lzov+Q?u^F*qHL!XZj*^{Vz zT)LwKV{)enCt*z-epmbO0>a0PcPTDkEQTfb!e4MsG2XcOx0?jC$bq<^KxKeL(nep z_F#bDjJ}Ixc2~O^VavUSZID@Xt{X~Ys?|DKg*pHNo zh8ItJ!hj;_>lsFi@@O|}XTt8wj;;l;yb+xxCIaH%+dCXjGnjv?+|suOXrA z4xVbq!#KO|fna{+-sZy?POF_j7IPLbpa@q`QuuoNy{JAZXG$?bywgN6tUkwVL_))X z=bze$#a~y{lG|w)zhI#Hz9F7BV(2Kf5q9hht<0(;oFK6+v{x<*|B8sgrJoSCf`{() z7#mNp=M}N}uVV zo<3N9bxi>%l3#KE66*bG^$sRwoH~5?z(K%vDkBbkBcQ;7t{uvGs0*j}_4~kj; zh77^s6f-~4urf^u1M9h2>3SpM9Y;@zhgbzPtH-<&tcP?7emgtH>na>MtSy<7du8r# z55z4ZfF77{nfsKvn;3w3j20EOEU>~025YO>l8oLBJ8>m&_w*;JOUjhQ=x(euj&Q2u zd}Eog^F@&tGdIij8<%STt)(oPUaGo9PpXQo8z@;6m(}wm+AX&{jyX_CWu`aODd!wz zBi?dDavzkN=5JIO&W>`VKG;?Xy$7yCa0~)us3hoH4y)Fc!#5!Jfoz%AAh%Ngrm3S@ z%`Q@9^D>ye37dyt2BrKlj=hWyTtRTcyAFg$|2PV&q~kLb7CopG;4bN_J>a5EouR zRm*{Ql6L7w%Su45?VDUUomeoUsE_^E6ZQ;s)RCN_IWYp^_^WSJfxT)}jLkd)?Yf}& zU?^hs-0Z{A$+~H)Eqn|Ha<~*#x_W$}xwd2$Ni56t&KnS=W5(@Uz`YeCXtG|2vjByS z2Zet{|w{9GTtaXF{^yVY#Ni9|l&s58WueCChq2pfyFE z(fpdUCh?pIN-Z3bPh0QLOh@cUmq9?E3!}Q{chYt6(j3oHFIG}1Sy6>sm{8X}30%f- zdW=6I&<13m^3tQZ)9hv2e_N&lGP2cE4o+@Xt6B#P4&KgBlBJpci!>SR6kf4Qh@(In zyOXuv@(hZ<#r07%x>39ErU(TgLYu*Okc?9o`<;l5N=n>DV(r9RDQy$8fX6>-&06A;EF%M|Tx8n#-um-T>vF+8UVk2x(_4#w4`37?Fo>?UAVr z!#vYzU5_W&IvqrhF1^WBg`0x5{thx`{*Mf8Sot^AO>}T^PQqU zNCQ6~DOl>f(97-?GDZ%CuU%dEy~NTBFvt+6n+`wOtkKg)<;|aG#5xZ&y+ybDXiA?( z_ai}qZ@O{Q;mrylOx;I zfc@42wrE%P&e+VWexG#$`wT35g44XiX@ZYA3 z@E#wi8ZL818VQvu&@oVjj!jxUkh7$fb%dC4SfV|#s`AJzwAEscp=FfOA7>QKiZsp& z?7$jhw%~=jiEGb;!WcCvu769zg8>PQGuXzNC?0-c-3Y2~`=37$s%9B<6bySyTkU`) z_V?2YCbm8}e#3$q+SQsuQ5P69yZrkT{GGkQvtOqfE|f2@S}AcHkj;KkPaw6zS$(|C zNxV(b2Z~99n%TG>fQon60xs_z1=0P3()sV`f>+6JVq_;t^7X zlj(d`wT>vsf2B$yEV|=mR`H*3rVXZ%ME1NY3i5;ug-ef7S$r05w8Dr(pzBs0{n4GO zxIg0u>17BT^-Qu;4UJNhw}``!jJ1j2trv*Wu5nsHHnCPI+MBqzuDrs5tjCb=eJZeW z!KdYiQeg0gHRNipjoU$N3Yc_AA8;!eM@7pED%=}&p;~x?@o-tZk|bkDE~*R2v72K; zG#0MuQj@elQ0O2uGOTOY>Z!sdbGCTQ*5$ViYOZd8FNF6kb_?sQqTM0RtRfRk?M=^m z3MO~N4UwJ{_so#AOXpcoXInzBHua1&(U^U{AkDSp0+sPp|KJgv**}Q*GhKbM!*`8-6Fzv%=Fr&CyUHL7HK+619T z26eAnM?>zRQGTE17V;98>PT5SibMBd)QEJuTJHUag5U>(@d+&t@f#bWui}Re{T3%! zAnq|ku@5FR9#zWhGjNJZ1vXkAZYRgf_d9LxsI$^JEBk$x6fPyf^1HO$IOGs>l;3Z` z;4lmL8UEF8hbGb`F?;_^A6gr1dw%-}{?$ zro#tyzVrCU>xH1&aGOu)nA_6_;*QHnssEdfpcK9?^yvrIs~td)D-W@t7S-8}+q?yi z|Mq^GMZf4DQbtvXY2#`_j$w6Ga+Ea^W3p!}4`W^2?*?N~r%F)d{-_$^a)E{%RHtbb z(1=g4t?wW#*d=KcDk-SHqKHix%n|gxw!*5zYwI`JUXKH|ytp@UCI9#L&6uov94aD6 z$iMQ7zqRR&n$Q8J_U~G&5cYZ!f;}IUy{F2MC#E-}2|G_*zdsYsinJvD= zJR9*HHp4bgU?Zh(Fay4*E0@F@gj{i#^kj5Tt(Cm%wq74p-*A1L5v_9aatiDP`Q8!u zD|)_^upp4jY9)l>5k5uyyHHBK7~r40;2?u*?SAZ=ek=mkY8A8hc(x0%N+(>C5;$0J{uqBape@=e4mn*!}%^qTfmJ*E#O{#qlsf=M@2@|a?J zO&fCs=@>|)7Sfo|0T)Ax)ofnM7&+#434$ScL6b^Bj9}9oe3q^K7}t8i99XKj8DxyJ z0D`KnFm*(L5eXyaQiH)%Z15@7^+Xuwqky|*8UG}0!OFsID#1{2^pSh}^wL>-&!r5b zc*?)nMY4eg?F3p58pg(fqDCTqXNFu)Dex zhMp;VVV($^M}$5~6}hW^hDOF;JruvAP8e}M@#TyE30M0O4!y!eJ}dHse5Gs&{{t@p z0)S-{a;;WGR3!8V0kX{T6;aLFl&4)!`t0R8MO<&%3znEIsQG`WSRG&;i#aFk9zHCf zNI_G2X`f}wC|~|h_O5w-WL(IH<=yc-_@6)iA(eHGVM1H~D;~w`iss@~P{$h#j@4)) z$Q=LP&h8bUL`UG6Fm#C6d)`eK2B(aR_XjxkDX=gJn-HicH2Hv_aCkyp1wmC0zOFby z$RC!TSTtAOY{{Y1(x`VWyE~k)gE@25t4`k-l*(u!#pVpF3Y@bgQ0D2R$=!1aT23=g z>Ez=)b&G1}6x{H>zb(wagIWW$ngxRUux6V6fmIdgl zl9EjAkr6bP{F(+3$VFC2`~wJ4+1?b_aPbJ}%_ZF|vj~j-%x8gB|I|dvH02A?=jJXc7fMzI%gn5&02AfqHJAS#Kr?@=BDSY`(RYl{4;0S=)#1 zk)Of5lJyK5DeNi4e&T=rOgrK2;ItxpEqb9qa=iU5J7COy187jt?Er_8k*T4KN&zs-Aam+ zd%{U}{*?c;`poR59%AohQb~^E0&S}W0zCUku`GVv)ps|Uv}KNrr(~M-gG?46a-9;h zqO1`vlN+Q0sZx^>U3y{q^~3+`;!MDy>fQi8u54k9Eg6$#EHTJZRLn$3*&<`jzSfkG zB}+A9%V#9YZ5d05k>ZQ8hdx5$BaF$ur^X;MeN!azpXRH+{Lg*vbKmzpzw_R6&vVc7 zoaZ_3d7b*>>)uq^Y>PoXiZQMd)6SaR}k-};DM!OQ^wX-Tq<54Ez1** zrPe+w&gWnMOVK5BRxbXC5N9xkJYDDHt-S6b&8LMjlJ1Wu%$OV7iE`@~d7OV&hI1;k zwxsl&T(06yzNDViAZWT)?#sjHX7h&?USU(<_9VUO@1P5w2Jqrjr=iYf)rTw33dpy@ z{2c1`HT%R6nUVg;XTAADFC&Aj4BZE9zwCAr%59-4J?t_p8|{^9*JP*;e>?uy=9K`i zz9E&A^uZ!Ml8n4thS1i{kjmMx$o;Ie2sc=VPnB5bmUnmAeJK=JPb|Gs!W@)T&UpfV03+jaF`#;^vGT>#pt~i`9@jYra{j zH%{p9y&WJYv6{ECbg_Yzm-+R_-TH(b78iCxdj=JgYM+m0p#zAkwF&W&4$Zrl&G>S?ube=kYi-Cxvw1K9i+}S4|1} zgRzTm^2G|N^^xXZGHpVu%j&`!{0B6B-rll3SrLHkJXq~dpaXAHN=s;VW>3`mMS@qG zo4+u-m?IzGFq=nSJblj`dDCa)|2ZzEz03YxT{=o7TC3_9B%1MOOsR@-O~8>F$D8Q| z+vzzTq>EwEXCC5<>rbPuq}K&Jq#Y^ff=j3Plcg!7V4^*}KWb+40fF<7_C zZM~9U*%q$2Pp!wPn(cyA7veh4qf2w?)pDMDK6k%=Mi4S^@pTXD>O1n{@0EQCb#6aS9uV$qFwiD8904Z~w+|#x)5tOM>o!r)D@tfyo4G=WGA) zNs2}uYm>|tXUI$9)}7kGNb@!n%@3WdHoiFCu8og#^nH6G&t;;^yN_bd)qj(L9rz-N zKYrM(W0k8+Y1u-u(RdpR9y!Y}n5D`%CE`LY$p{r5@wZ zP;au>HAB}xLsu%%jrG*YailpeIi$lTsg7`3#qqTtRy3IW=~a-4<-{|ku}$jJhSK!& zjn|{{FNJJE7Dv~b#|#>lB$BR$Z1NwJ;(DP}>-qkpLj4ZZ>VVOclp3l`XQicQL;tC~ zCl!@}@cp$Ob;1ke{Tw$m9%O|&?kwku_}JJI5~6RlVxvYK*+W3Cd!NWXe>1&5Vb*?y zFif$y$bYG`yuVzcHHw;-GVJmd=6r8|w=!lT=7PXf`~g7@nS6VxH;Ri+3C$j8V>k`#uq%~6(pT@dV{CV6s4od{>EXi(mx z*gmVu80CA2BRr6uhDS@{d4Qw%g!E!pzEc}MBWqiEBEf* z5w^4wSv0mEb3FfnKQy^E2V+Xil2dgKyD=$Luegx&$Yr5LaWL~3{0?Iq!TICjHy$@8 zdWI({XeCOogA^}TBs#26M4Je0@kKkEL3wgYkpqwMNoejUNd*Hx)*7=R1r{Rj5+BYT z@6_$L}ElNS%T@`BvY^W-3Lx)hsT!c;I1KUYEY1!H1hvKUX%q{R0&O&eLFgC?aNNgs zVLc{;V8?&55~n1JKp@C1Z4T?Opof*8~XmY&FJ3fZc2Mnv??q`M?SUIk2UjsuLT8c?yCD-FJf9Ct!#zi=-_u^yCj% z6m*|J0YtBV-#Kv<$na;28xtnLP23NW2lSXc0DK4mGbXJ8>rilh(jFj%f`_N{ftR6R zY}8(`cNzvhpMnFn7g(w!_<8C$z-riS?V$H3k?Q_k$!jBp3de9NeJ-TTNLn523zgTl>(h-VIa>tIPftWd{U?aTD~&` zR&rV5LAIAcRt--9`22w$s7^xx+X~rG98DL1JYb76G(+G+G22_$V_}wa7>JvN1ICZo zCfN#Uwn?S>_s(2?%&J6!VsuSluo=A3t_%j!QNY{3St1f_>iu^>hUjpBw;N>A{eiI_ z&~*+2strSx|9@8gH)Lg$EpznYq!!})r2JXPu^fF{fVIh8VqC0XA_zp6bX;T&e Gf&33iR&0&{ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a4b44297..e750102e 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 2fe81a7d..1b6c7873 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ -#!/usr/bin/env sh +#!/bin/sh # -# Copyright 2015 the original author or authors. +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,78 +17,113 @@ # ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` +APP_BASE_NAME=${0##*/} # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -97,7 +132,7 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" + JAVACMD=java which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the @@ -105,79 +140,95 @@ location of your Java installation." fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=`expr $i + 1` + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat index 9109989e..ac1b06f9 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -40,7 +40,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init +if "%ERRORLEVEL%" == "0" goto execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. @@ -54,7 +54,7 @@ goto fail set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe -if exist "%JAVA_EXE%" goto init +if exist "%JAVA_EXE%" goto execute echo. echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% @@ -64,28 +64,14 @@ echo location of your Java installation. goto fail -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - :execute @rem Setup the command line set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* :end @rem End local scope for the variables with windows NT shell From 67266487c05c26ea2e8bac36b25ba549cf261c4d Mon Sep 17 00:00:00 2001 From: avikganguly01 Date: Mon, 29 May 2023 01:28:09 +0530 Subject: [PATCH 059/156] BootJar --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6f7e212f..17de7894 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,7 +23,7 @@ jobs: # - slack/notify: # event: pass # template: basic_success_1 - - run: ./gradlew build + - run: ./gradlew clean bootJar # - run: ./gradlew cucumberCli # run tests! Slack Success/Fail Notification Step #- run: ./gradlew test From 099ac1a8a43fd16b39ee33579edd7f19696e28e7 Mon Sep 17 00:00:00 2001 From: avikganguly01 Date: Mon, 29 May 2023 01:30:27 +0530 Subject: [PATCH 060/156] Removing gradle from inside Dockerfile --- .circleci/config.yml | 2 -- Dockerfile | 1 - 2 files changed, 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 17de7894..3ff31c10 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,6 @@ jobs: build: docker: - image: cimg/openjdk:17.0.0 - - image: docker:17.05.0-ce-git working_directory: ~/repo environment: # Customize the JVM maximum heap limit @@ -14,7 +13,6 @@ jobs: TERM: dumb steps: - checkout - - setup_remote_docker # - slack/notify: # event: fail diff --git a/Dockerfile b/Dockerfile index 31018ee7..4bad37e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ FROM openjdk:17 EXPOSE 5000 -./gradlew build COPY build/libs/*.jar . CMD java -jar *.jar From 8a1aadf6448c4af6ceba2c56200b8ff782a4a136 Mon Sep 17 00:00:00 2001 From: avikganguly01 Date: Mon, 29 May 2023 01:36:37 +0530 Subject: [PATCH 061/156] Sequential Docker Push --- .circleci/config.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3ff31c10..751410fb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,16 +21,7 @@ jobs: # - slack/notify: # event: pass # template: basic_success_1 - - run: ./gradlew clean bootJar - # - run: ./gradlew cucumberCli - # run tests! Slack Success/Fail Notification Step - #- run: ./gradlew test -workflows: - build_and_push_image: - jobs: - - build: - context: - - slack + - run: ./gradlew clean bootJar - aws-ecr/build-and-push-image: aws-access-key-id: AWS_ACCESS_KEY_ID aws-secret-access-key: AWS_SECRET_ACCESS_KEY @@ -42,4 +33,14 @@ workflows: repo: phee-bulk-processor repo-scan-on-push: true role-arn: arn:aws:iam::419830066942:role/CustomAdmin - tag: latest + tag: latest + # - run: ./gradlew cucumberCli + # run tests! Slack Success/Fail Notification Step + #- run: ./gradlew test +workflows: + build_and_push_image: + jobs: + - build: + context: + - slack + From 1ccb40140d04b95707517c317492f1698fa04e52 Mon Sep 17 00:00:00 2001 From: avikganguly01 Date: Mon, 29 May 2023 01:37:56 +0530 Subject: [PATCH 062/156] Passing context from workflow --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 751410fb..b01a9f51 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ jobs: - aws-ecr/build-and-push-image: aws-access-key-id: AWS_ACCESS_KEY_ID aws-secret-access-key: AWS_SECRET_ACCESS_KEY - context: AWS + # context: AWS extra-build-args: '--compress' push-image: true region: ap-south-1 @@ -42,5 +42,6 @@ workflows: jobs: - build: context: + - AWS - slack From b7ffdc5dc77d46df0ed48145fa836ea89220010a Mon Sep 17 00:00:00 2001 From: avikganguly01 Date: Mon, 29 May 2023 01:40:12 +0530 Subject: [PATCH 063/156] Remote Docker --- .circleci/config.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b01a9f51..88e78155 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,6 +6,7 @@ jobs: build: docker: - image: cimg/openjdk:17.0.0 + - image: docker:17.05.0-ce-git working_directory: ~/repo environment: # Customize the JVM maximum heap limit @@ -13,19 +14,10 @@ jobs: TERM: dumb steps: - checkout - -# - slack/notify: -# event: fail -# mentions: '@here' -# template: basic_fail_1 -# - slack/notify: -# event: pass -# template: basic_success_1 - - run: ./gradlew clean bootJar + - setup_remote_docker - aws-ecr/build-and-push-image: aws-access-key-id: AWS_ACCESS_KEY_ID aws-secret-access-key: AWS_SECRET_ACCESS_KEY - # context: AWS extra-build-args: '--compress' push-image: true region: ap-south-1 @@ -33,7 +25,16 @@ jobs: repo: phee-bulk-processor repo-scan-on-push: true role-arn: arn:aws:iam::419830066942:role/CustomAdmin - tag: latest + tag: latest + +# - slack/notify: +# event: fail +# mentions: '@here' +# template: basic_fail_1 +# - slack/notify: +# event: pass +# template: basic_success_1 + - run: ./gradlew clean bootJar # - run: ./gradlew cucumberCli # run tests! Slack Success/Fail Notification Step #- run: ./gradlew test From 6ecb4c273ebf32422b590acf0e8f54c14c44839c Mon Sep 17 00:00:00 2001 From: avikganguly01 Date: Mon, 29 May 2023 01:40:53 +0530 Subject: [PATCH 064/156] Remote Docker --- .circleci/config.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 88e78155..14165c20 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,6 +15,14 @@ jobs: steps: - checkout - setup_remote_docker +# - slack/notify: +# event: fail +# mentions: '@here' +# template: basic_fail_1 +# - slack/notify: +# event: pass +# template: basic_success_1 + - run: ./gradlew clean bootJar - aws-ecr/build-and-push-image: aws-access-key-id: AWS_ACCESS_KEY_ID aws-secret-access-key: AWS_SECRET_ACCESS_KEY @@ -25,16 +33,7 @@ jobs: repo: phee-bulk-processor repo-scan-on-push: true role-arn: arn:aws:iam::419830066942:role/CustomAdmin - tag: latest - -# - slack/notify: -# event: fail -# mentions: '@here' -# template: basic_fail_1 -# - slack/notify: -# event: pass -# template: basic_success_1 - - run: ./gradlew clean bootJar + tag: latest # - run: ./gradlew cucumberCli # run tests! Slack Success/Fail Notification Step #- run: ./gradlew test From f0b41ec2585557d185653adf22bd17b5c9305f95 Mon Sep 17 00:00:00 2001 From: Avik Ganguly Date: Mon, 29 May 2023 03:25:19 +0530 Subject: [PATCH 065/156] Deleting GitActions --- .github/pull_request_template.md | 20 -------------------- .github/workflows/test.yml | 12 ------------ 2 files changed, 32 deletions(-) delete mode 100644 .github/pull_request_template.md delete mode 100644 .github/workflows/test.yml diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index 98c407da..00000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,20 +0,0 @@ -## Description - -* Describe the changes made and why they were made. -* Add a link to teh design document or include the design bullet points related to this PR here. - - _(Ignore if these details are present on the associated JIRA ticket)_ - -## Checklist - -Please make sure these boxes are checked before submitting your pull request - thanks! - -- [ ] Design related bullet points or design document link related to this PR added in the description above. - -- [ ] Updated corresponding Postman Collection or Api documentation for the changes in this PR. - -- [ ] Create/update unit or integration tests for verifying the changes made. - -- [ ] Add required Swagger annotation and update API documentation with details of any API changes if applicable - -- [ ] Followed the naming conventions as given in https://docs.google.com/document/d/1Q4vaMSzrTxxh9TS0RILuNkSkYCxotuYk1Xe0CMIkkCU/edit?usp=sharing diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index f71e4d7d..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: Test -on: [push, pull_request] -jobs: - Assemble: - runs-on: ubuntu-latest - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - name: Build project - run: ./gradlew clean build - - name: Build Docker image - run: docker build . -t paymenthubee.mifos.io/phee/bulk-processor From 98a0d0ea78baa1841bcbc60e7225e7ac16041be5 Mon Sep 17 00:00:00 2001 From: Arka Das <127482132+arkadasfynarfin@users.noreply.github.com> Date: Thu, 1 Jun 2023 12:42:22 +0530 Subject: [PATCH 066/156] upgrade zeebe to 8.1.1 (#54) Co-authored-by: Manoj <56669674+fynmanoj@users.noreply.github.com> --- build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a8b8e68a..8e709c33 100644 --- a/build.gradle +++ b/build.gradle @@ -39,7 +39,8 @@ dependencies { implementation 'com.amazonaws:aws-java-sdk-dynamodb:1.11.486' implementation 'com.azure:azure-storage-blob:12.12.0' - implementation 'io.camunda:zeebe-client-java:1.1.0' + + implementation 'io.camunda:zeebe-client-java:8.1.1' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.12.3' implementation 'org.springframework.kafka:spring-kafka:2.5.8.RELEASE' From fbb886a7ce7ee30fcaeffdc5c553f3d750cfce2b Mon Sep 17 00:00:00 2001 From: Ishan Ranasingh <44061541+truthfool@users.noreply.github.com> Date: Thu, 1 Jun 2023 12:44:49 +0530 Subject: [PATCH 067/156] Added filter for logging tenant (#57) * removed optional * removed check --- .../java/org/mifos/processor/bulk/api/ApiOriginFilter.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java b/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java index 2f849267..159c0e36 100644 --- a/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java +++ b/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java @@ -27,11 +27,7 @@ public void init(FilterConfig filterConfig) throws ServletException { public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { HttpServletRequest req = (HttpServletRequest) request; String tenant = req.getHeader("Platform-TenantId"); - Optional tenantVal = Optional.of(tenant); - if (tenantVal.isPresent()) { - logger.info("Tenant Name is : {}", tenant); - } - + logger.debug("Tenant Name is : {}", tenant); } @Override From 4d6e9cde14d68bee775ca4beffb776b815693677 Mon Sep 17 00:00:00 2001 From: Ankita Ranjan <92036361+ankita10r@users.noreply.github.com> Date: Thu, 1 Jun 2023 12:48:11 +0530 Subject: [PATCH 068/156] GOV-75 (#64) --- .../bulk/camel/routes/InitSubBatchRoute.java | 1 + .../processor/bulk/config/PaymentModeMapping.java | 2 +- src/main/resources/application-paymentmode.yaml | 15 +++++++++++++++ src/main/resources/application.yaml | 12 +----------- 4 files changed, 18 insertions(+), 12 deletions(-) create mode 100644 src/main/resources/application-paymentmode.yaml diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index e8639822..7230f013 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -104,6 +104,7 @@ public void configure() throws Exception { PaymentModeMapping mapping = paymentModeConfiguration.getByMode(paymentMode); String tenantName = exchange.getProperty(TENANT_NAME, String.class); + tenantName = mapping.getDebulkingDfspid()==null ? tenantName : mapping.getDebulkingDfspid(); Map variables = exchange.getProperty(ZEEBE_VARIABLE, Map.class); variables.put(PAYMENT_MODE, paymentMode); zeebeProcessStarter.startZeebeWorkflow( diff --git a/src/main/java/org/mifos/processor/bulk/config/PaymentModeMapping.java b/src/main/java/org/mifos/processor/bulk/config/PaymentModeMapping.java index 886a8c28..5895b52c 100644 --- a/src/main/java/org/mifos/processor/bulk/config/PaymentModeMapping.java +++ b/src/main/java/org/mifos/processor/bulk/config/PaymentModeMapping.java @@ -7,6 +7,6 @@ @Setter public class PaymentModeMapping { - private String id, endpoint; + private String id, endpoint, debulkingDfspid; private PaymentModeType type; } diff --git a/src/main/resources/application-paymentmode.yaml b/src/main/resources/application-paymentmode.yaml new file mode 100644 index 00000000..3159888a --- /dev/null +++ b/src/main/resources/application-paymentmode.yaml @@ -0,0 +1,15 @@ +payment-mode: + mappings: + - id: "GSMA" + type: "PAYMENT" + endpoint: "/channel/gsma/transfer" + - id: "MOJALOOP" + type: "PAYMENT" + endpoint: "/channel/transfer" + - id: "SLCB" + type: "BULK" + endpoint: "bulk_connector_{MODE}-{dfspid}" + - id: "CLOSEDLOOP" + type: "BULK" + endpoint: "bulk_connector_{MODE}-{dfspid}" + debulkingDfspid: "lion" \ No newline at end of file diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 6040a243..f062b90c 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -75,17 +75,7 @@ bpmn: bulk-processor: "bulk_processor-{dfspid}" slcb: "slcb-{dfspid}" -payment-mode: - mappings: - - id: "GSMA" - type: "PAYMENT" - endpoint: "/channel/gsma/transfer" - - id: "MOJALOOP" - type: "PAYMENT" - endpoint: "/channel/transfer" - - id: "SLCB" - type: "BULK" - endpoint: "bulk_connector_{MODE}-{dfspid}" +#payment mode moved to a separate application property file for helm rewrite config: minimum-successful-tx-ratio: 0.90 From 39239cccec0138079d8427e5f3bd7488e0d1538e Mon Sep 17 00:00:00 2001 From: Ankita Ranjan <92036361+ankita10r@users.noreply.github.com> Date: Thu, 1 Jun 2023 18:45:49 +0530 Subject: [PATCH 069/156] Batch files go through a final check to be clean of defects and inconsistencies (#61) --- build.gradle | 2 + .../camel/routes/ProcessorStartRoute.java | 219 +++++++++++++++--- .../processor/bulk/zeebe/ZeebeVariables.java | 4 + src/main/resources/application.yaml | 7 +- 4 files changed, 205 insertions(+), 27 deletions(-) diff --git a/build.gradle b/build.gradle index 8e709c33..e60e790b 100644 --- a/build.gradle +++ b/build.gradle @@ -46,6 +46,8 @@ dependencies { implementation 'org.apache.camel.springboot:camel-mail-starter:3.4.0' implementation 'org.projectlombok:lombok:1.18.24' + implementation 'org.apache.tika:tika-core:1.4' + implementation 'org.apache.commons:commons-io:1.3.2' annotationProcessor 'org.projectlombok:lombok:1.18.24' testImplementation 'org.apache.camel:camel-test-spring-junit5:3.4.0' diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 3c342cda..85685d60 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -2,6 +2,9 @@ import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; + +import org.apache.tika.Tika; + import org.json.JSONObject; import org.mifos.processor.bulk.file.FileTransferService; import org.mifos.processor.bulk.utility.PhaseUtils; @@ -14,12 +17,13 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import java.io.File; -import java.io.FileWriter; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; + +import javax.activation.DataHandler; +import javax.mail.internet.MimeBodyPart; +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.stream.Collectors; import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_REQUEST_TYPE; import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; @@ -60,6 +64,12 @@ public class ProcessorStartRoute extends BaseRouteBuilder { @Value("${pollingApi.timer}") private String pollApiTimer; + @Value("#{'${csv.columnNames}'.split(',')}") + private List columnNames; + + @Value("${csv.size}") + private int csvSize; + private final Logger logger = LoggerFactory.getLogger(this.getClass()); @Autowired @@ -75,14 +85,28 @@ private void setup() { from("rest:POST:/batchtransactions") .id("rest:POST:/batchtransactions") .log("Starting route rest:POST:/batchtransactions") + .to("direct:validate-file") + .choice() + .when(header("CamelHttpResponseCode").isNotEqualTo("200")) + .log(LoggingLevel.ERROR, "File upload failed") + .otherwise() .unmarshal().mimeMultipart("multipart/*") .process(exchange -> { String batchId = UUID.randomUUID().toString(); exchange.setProperty(BATCH_ID,batchId); }) + .unmarshal().mimeMultipart("multipart/*") + .to("direct:validateFileSyncResponse") + .choice() + .when(header("CamelHttpResponseCode").isNotEqualTo("200")) + .log(LoggingLevel.ERROR, "File upload failed") + .otherwise() + .unmarshal().mimeMultipart("multipart/*") .wireTap("direct:executeBatch") - .to("direct:pollingOutput"); + .to("direct:pollingOutput") + .endChoice() + .endChoice(); @@ -124,26 +148,25 @@ private void setup() { String requestId = exchange.getProperty(REQUEST_ID, String.class); String purpose = exchange.getProperty(PURPOSE, String.class); String batchId = exchange.getProperty(BATCH_ID,String.class); + String note = null; if (purpose == null || purpose.isEmpty()) { purpose = "test payment"; } - logger.info("\n\n Filename: " + fileName + " \n\n"); - logger.info("\n\n BatchId: " + batchId + " \n\n"); - - File file = new File(fileName); - file.setWritable(true); - file.setReadable(true); - - String csvData = exchange.getIn().getBody(String.class); - FileWriter fileWriter = new FileWriter(file); - fileWriter.write(csvData); - fileWriter.close(); - - logger.info(""+file.length()); - logger.info(file.getAbsolutePath()); + logger.info("\n\n Filename: {}", fileName); + logger.info("\n\n BatchId: {} ",batchId); + + InputStream csvData = exchange.getIn().getBody(InputStream.class); + File file = setupFile(csvData,fileName); + logger.debug("File length {}",file.length()); + boolean verifyData = verifyData(file); + logger.debug("Data verification result {}", verifyData); + if(!verifyData){ + note = "Invalid data in file data processing stopped"; + + } String nm = fileTransferService.uploadFile(file, bucketName); @@ -154,8 +177,9 @@ private void setup() { exchange.setProperty(CALLBACK_URL,callbackUrl); List phases = phaseUtils.getValues(); - logger.info(phases.toString()); + logger.debug(phases.toString()); Map variables = new HashMap<>(); + variables.put(FILE_VALIDITY, verifyData); variables.put(BATCH_ID, batchId); variables.put(FILE_NAME, fileName); variables.put(REQUEST_ID, requestId); @@ -164,6 +188,7 @@ private void setup() { variables.put(CALLBACK_URL,callbackUrl); variables.put(PHASES,phases); variables.put(PHASE_COUNT,phases.size()); + variables.put(NOTE,note); setConfigProperties(variables); JSONObject response = new JSONObject(); @@ -233,18 +258,160 @@ private void setup() { .id("direct:pollingOutput") .log("Started pollingOutput route") .process(exchange -> { -// String response = "PollingPath: /batch/Summary/{batchId} and SuggestedCallbackSeconds: # seconds"; -// response = response.replace("#",pollApiTimer).replace("{batchId}", (CharSequence) exchange.getProperty(BATCH_ID)); -// ObjectMapper mapper = new ObjectMapper(); JSONObject json = new JSONObject(); json.put("PollingPath", "/batch/Summary/"+exchange.getProperty(BATCH_ID)); json.put("SuggestedCallbackSeconds",pollApiTimer); exchange.getIn().setBody(json.toString()); - }); + from("direct:validateFileSyncResponse") + .id("direct:validateFileSyncResponse") + .log("Starting route direct:validateFileSyncResponse") + .process(exchange -> { + String fileName = System.currentTimeMillis() + "_" + exchange.getIn().getHeader("fileName", String.class); + InputStream csvData = exchange.getIn().getBody(InputStream.class); + File file = setupFile(csvData,fileName); + int fileSize = (int) file.length(); + if (fileSize > csvSize) { + setErrorResponse(exchange, 400, "File too big", + "The file uploaded is too big. " + + "Please upload a file and try again."); + } + else if(!verifyCsv(file)){ + setErrorResponse(exchange, 400, "Invalid file structure", + "The file uploaded contains wrong structure." + + " Please upload correct file columns and try again."); + } + else { + logger.debug("Filename: {}", fileName); + setResponse(exchange, 200); + } + + }) + .log("Completed route direct:validateFileSyncResponse"); + + from("direct:validate-file") + .id("direct:validate-file") + .log("Starting route direct:validate-file") + .process(exchange -> { + System.setProperty("mail.mime.multipart.ignoreexistingboundaryparameter", "true"); + System.setProperty("mail.mime.multipart.allowempty", "true"); + InputStream inputStreams = exchange.getIn().getBody(InputStream.class); + MimeBodyPart mimeMessage = new MimeBodyPart(inputStreams); + DataHandler dh = mimeMessage.getDataHandler(); + logger.debug("File name: {} ",dh.getName()); + Tika tika = new Tika(); + String fileType = tika.detect(dh.getName()); + logger.debug("File type: {} ", fileType); + if (dh.getName() == null || dh.getName().isEmpty() ) { + setErrorResponse(exchange, 400, "File not uploaded", + "There was no fie uploaded with the request. " + + "Please upload a file and try again."); + } + else if (!fileType.equalsIgnoreCase("text/csv")) { + setErrorResponse(exchange, 400, "Broken file", + "The file uploaded is broken as it has a different extension. " + + "Please upload a csv file and try again."); + } + else{ + setResponse(exchange, 200);} + + });} + + private boolean verifyData(File file) throws IOException { + InputStream ips = new FileInputStream(file); + InputStreamReader ipsr = new InputStreamReader(ips); + BufferedReader br = new BufferedReader(ipsr); + String line; + br.readLine(); + while ((line = br.readLine()) != null) { + String[] row = line.split(","); + if (row.length != columnNames.size()) { + logger.debug("Row invalid {} {}", row.length, columnNames.size()); + return false; + } + if(!verifyRow(row)) + return false; + } + return true; } + private boolean verifyRow(String[] row) { + for (int i = 1; i < row.length; i++) { + row[i] = row[i].trim(); + if (row[i].equalsIgnoreCase("MSISDN")) { + int j = row[i].indexOf("MSISDN"); + if (!(j == row.length) ){ + if(!row[j+ 1].matches("^[0-9]*$")) { + logger.debug("MSISDN invalid"); + return false; + } + } + } + else if (row[i].contains("amount")){ + int j = row[i].indexOf("amount"); + if(!row[j].matches("^[0-9]*$")){ + logger.debug("Amount invalid"); + return false; + } + + } + } + return true; + } + + private File setupFile(InputStream csvStream, String fileName) throws IOException { + String csvData = new BufferedReader( + new InputStreamReader(csvStream, StandardCharsets.UTF_8)) + .lines() + .collect(Collectors.joining("\n")); + File file = new File(fileName); + file.setWritable(true); + file.setReadable(true); + FileWriter fileWriter = new FileWriter(file); + fileWriter.write(csvData); + fileWriter.close(); + return file; + } + + + private boolean verifyCsv(File csvData) throws IOException { + BufferedReader br = new BufferedReader(new FileReader(csvData)); + String header = br.readLine(); + String[] columns = new String[0]; + if (header != null) { + columns = header.split(","); + logger.info("Columns in the csv file are {}", Arrays.toString(columns)); + } + int i = 0; + while(i < columns.length) { + if(columnNames.contains(columns[i])) { + logger.debug("Column name {} is at index {} ", columns[i], columnNames.indexOf(columns[i])); + i++; + + } else { + return false; + } + } + return true; + } + + private void setErrorResponse(Exchange exchange,int responseCode, String errorInfo, String errorDescription) { + // TODO Auto-generated method stub + JSONObject json = new JSONObject(); + json.put("Error Information: ", errorInfo); + json.put("Error Description : ",errorDescription); + exchange.getIn().setHeader(Exchange.HTTP_RESPONSE_CODE, responseCode); + exchange.getIn().setBody(json.toString()); + exchange.setProperty("body", json); + logger.error("Error response is {}", json); + } + private void setResponse(Exchange exchange,int responseCode){ + exchange.getIn().setHeader(Exchange.HTTP_RESPONSE_CODE, responseCode); + + } + + private Map setConfigProperties(Map variables) { variables.put(PARTY_LOOKUP_ENABLED, workerConfig.isPartyLookUpWorkerEnabled); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 2bdf2d17..ee5f45fe 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -114,4 +114,8 @@ private ZeebeVariables() { public static final String PHASES = "phases"; public static final String PHASE_COUNT = "phaseCount"; + + public static final String FILE_VALIDITY = "isFileValid"; + + public static final String NOTE = "note"; } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index f062b90c..300f3ef9 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -115,4 +115,9 @@ callback-phases: - 40 - 60 - 80 - - 100 \ No newline at end of file + - 100 + + +csv: + columnNames: "id,request_id,payment_mode,payer_identifier_type,payer_identifier,payee_identifier_type,payee_identifier,amount,currency,note" + size : 100000 # in bytes \ No newline at end of file From 857b10220101ef7008e5082580605f178147acd6 Mon Sep 17 00:00:00 2001 From: Ankita Ranjan <92036361+ankita10r@users.noreply.github.com> Date: Thu, 1 Jun 2023 19:32:22 +0530 Subject: [PATCH 070/156] GOV-75-fixed debulking dfspid logic (#69) * GOV-75-fixed debulking dfspid logic * Merge branch 'GOV75' of /Users/ankitaranjan/Desktop/PHEE/ph-ee-bulk-processor with conflicts. --------- Co-authored-by: Manoj <56669674+fynmanoj@users.noreply.github.com> --- .../mifos/processor/bulk/camel/routes/InitSubBatchRoute.java | 2 ++ .../java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index 7230f013..1a3eb532 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -107,6 +107,8 @@ public void configure() throws Exception { tenantName = mapping.getDebulkingDfspid()==null ? tenantName : mapping.getDebulkingDfspid(); Map variables = exchange.getProperty(ZEEBE_VARIABLE, Map.class); variables.put(PAYMENT_MODE, paymentMode); + variables.put(DEBULKINGDFSPID, + mapping.getDebulkingDfspid()==null ? tenantName : mapping.getDebulkingDfspid()); zeebeProcessStarter.startZeebeWorkflow( Utils.getBulkConnectorBpmnName(mapping.getEndpoint(), mapping.getId().toLowerCase(), tenantName), variables); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index ee5f45fe..9c0dd312 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -115,7 +115,10 @@ private ZeebeVariables() { public static final String PHASE_COUNT = "phaseCount"; + public static final String DEBULKINGDFSPID = "debulkingDfspid"; + public static final String FILE_VALIDITY = "isFileValid"; public static final String NOTE = "note"; + } From 75c1d249403a7ce0e7f2463fbf8ef3b941a580e7 Mon Sep 17 00:00:00 2001 From: Apurb Rajdhan <103931815+apurbraj@users.noreply.github.com> Date: Thu, 1 Jun 2023 23:19:49 +0530 Subject: [PATCH 071/156] GOV-327 (#66) * GOV-327 --- .../bulk/camel/config/CamelProperties.java | 7 ++ .../AccountLookupCallbackProcessor.java | 58 +++++++++++++ .../bulk/camel/routes/AccountLookupRoute.java | 46 ++++++++++ .../camel/routes/ExternalApiCallRoute.java | 30 +++++++ .../bulk/schema/AccountLookupResponseDTO.java | 20 +++++ .../bulk/schema/PaymentModalityDTO.java | 16 ++++ .../processor/bulk/zeebe/ZeebeMessages.java | 1 + .../processor/bulk/zeebe/ZeebeVariables.java | 8 ++ .../worker/AccountLookupCallbackWorker.java | 50 +++++++++++ .../zeebe/worker/AccountLookupWorker.java | 85 +++++++++++++++++++ .../processor/bulk/zeebe/worker/Worker.java | 4 +- src/main/resources/application.yaml | 9 +- 12 files changed, 331 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/camel/processor/AccountLookupCallbackProcessor.java create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java create mode 100644 src/main/java/org/mifos/processor/bulk/schema/AccountLookupResponseDTO.java create mode 100644 src/main/java/org/mifos/processor/bulk/schema/PaymentModalityDTO.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 4201adc3..3977125a 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -62,5 +62,12 @@ private CamelProperties() {} public static final String PAYLOAD = "payload"; public static final String BATCH_ID_HEADER = "X-BatchID"; + public static final String HOST = "externalApiCallHost"; + public static final String ENDPOINT = "externalApiCallEndpoint"; + public static final String CACHED_TRANSACTION_ID = "cachedTransactionId"; + public static final String PAYEE_IDENTITY = "payeeIdentity"; + public static final String PAYMENT_MODALITY = "paymentModality"; + public static final String PAYEE_PARTY_ID = "payeePartyId"; + public static final String PAYEE_PARTY_ID_TYPE = "payeePartyIdType"; } diff --git a/src/main/java/org/mifos/processor/bulk/camel/processor/AccountLookupCallbackProcessor.java b/src/main/java/org/mifos/processor/bulk/camel/processor/AccountLookupCallbackProcessor.java new file mode 100644 index 00000000..3b2f95be --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/processor/AccountLookupCallbackProcessor.java @@ -0,0 +1,58 @@ +package org.mifos.processor.bulk.camel.processor; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.zeebe.client.ZeebeClient; +import org.apache.camel.Exchange; +import org.apache.camel.Processor; +import org.mifos.processor.bulk.schema.AccountLookupResponseDTO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.zeebe.ZeebeMessages.ACCOUNT_LOOKUP; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; + +@Component +public class AccountLookupCallbackProcessor implements Processor { + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired(required = false) + private ZeebeClient zeebeClient; + @Autowired + private ObjectMapper objectMapper; + @Override + public void process(Exchange exchange) throws Exception { + Map variables = new HashMap<>(); + + String error = null; + String response = exchange.getIn().getBody(String.class); + AccountLookupResponseDTO accountLookupResponseDTO = null; + try { + accountLookupResponseDTO = objectMapper.readValue(response, AccountLookupResponseDTO.class); + variables.put(ACCOUNT_LOOKUP_FAILED, false); + variables.put(PAYEE_PARTY_ID, accountLookupResponseDTO.getPaymentModalityList().get(0).getFinancialAddress()); + variables.put(PAYEE_PARTY_ID_TYPE, accountLookupResponseDTO.getPaymentModalityList().get(0).getPaymentModality()); + exchange.setProperty(CACHED_TRANSACTION_ID, accountLookupResponseDTO.getRequestId()); + }catch(IOException e){ + variables.put(ACCOUNT_LOOKUP_FAILED, true); + error = objectMapper.readValue(response, String.class); + } + + if(zeebeClient != null) { + + zeebeClient.newPublishMessageCommand() + .messageName(ACCOUNT_LOOKUP) + .correlationKey(exchange.getProperty(CACHED_TRANSACTION_ID, String.class)) + .timeToLive(Duration.ofMillis(50000)) + .variables(variables) + .send(); + } + } +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java new file mode 100644 index 00000000..19edbdd8 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java @@ -0,0 +1,46 @@ +package org.mifos.processor.bulk.camel.routes; + +import org.apache.camel.Exchange; +import org.apache.camel.LoggingLevel; +import org.mifos.processor.bulk.camel.config.CamelProperties; +import org.mifos.processor.bulk.camel.processor.AccountLookupCallbackProcessor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; + +@Component +public class AccountLookupRoute extends BaseRouteBuilder{ + @Autowired + AccountLookupCallbackProcessor accountLookupCallbackProcessor; + @Override + public void configure() throws Exception { + + from("rest:POST:/accountLookup/Callback") + .log(LoggingLevel.DEBUG, "######## -> ACCOUNT LOOKUP CALLBACK") + .process(accountLookupCallbackProcessor) + .setBody(constant("Received")) + .setHeader(Exchange.HTTP_RESPONSE_CODE, constant(200)) + .end(); + + from("direct:send-account-lookup") + .id("account-lookup") + .process(exchange -> { + String callbackUrl = exchange.getProperty(CALLBACK, String.class); + exchange.getIn().setHeader(CALLBACK, callbackUrl); + }) + .setHeader(Exchange.HTTP_METHOD, constant("GET")) + .setHeader( + Exchange.HTTP_QUERY, + simple(new StringBuilder().append(PAYEE_IDENTITY).append("=${exchangeProperty.") + .append(PAYEE_IDENTITY).append("}&").append(PAYMENT_MODALITY) + .append("=${exchangeProperty.").append(PAYMENT_MODALITY) + .append("}&").append("requestId=${exchangeProperty.requestId}").toString() + ) + ) + .setProperty(HOST, simple("{{identity-account-mapper.hostname}}")) + .setProperty(ENDPOINT, simple("identityAccountMapper/accountLookup/")) + .to("direct:external-api-calling"); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java new file mode 100644 index 00000000..8f549ed8 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java @@ -0,0 +1,30 @@ +package org.mifos.processor.bulk.camel.routes; + +import org.apache.camel.LoggingLevel; +import org.apache.camel.builder.RouteBuilder; +import org.springframework.stereotype.Component; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.ENDPOINT; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HOST; + +@Component +public class ExternalApiCallRoute extends RouteBuilder { + @Override + public void configure() { + from("direct:external-api-calling") + .id("external-api-call") + .log(LoggingLevel.DEBUG,"######## API CALL -> Calling an external api") + .process(exchange -> { + // remove the trailing "/" from endpoint + String endpoint = exchange.getProperty(ENDPOINT, String.class); + if (endpoint.startsWith("/")) { exchange.setProperty(ENDPOINT, endpoint.substring(1)); } + }) + .log(LoggingLevel.DEBUG,"Host: ${exchangeProperty." + HOST + "}") + .log(LoggingLevel.DEBUG,"Endpoint: ${exchangeProperty." + ENDPOINT + "}") + .log(LoggingLevel.DEBUG,"Headers: ${headers}") + .log(LoggingLevel.DEBUG,"Request Body: ${body}") + .toD("${exchangeProperty." + HOST + "}/${exchangeProperty." + ENDPOINT + "}" + + "?bridgeEndpoint=true" + "&throwExceptionOnFailure=false") + .log(LoggingLevel.DEBUG,"Response body: ${body}"); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/schema/AccountLookupResponseDTO.java b/src/main/java/org/mifos/processor/bulk/schema/AccountLookupResponseDTO.java new file mode 100644 index 00000000..5427d135 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/AccountLookupResponseDTO.java @@ -0,0 +1,20 @@ +package org.mifos.processor.bulk.schema; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.io.Serializable; +import java.util.List; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class AccountLookupResponseDTO implements Serializable { + private String requestId; + private String payeeIdentity; + private List paymentModalityList; + +} diff --git a/src/main/java/org/mifos/processor/bulk/schema/PaymentModalityDTO.java b/src/main/java/org/mifos/processor/bulk/schema/PaymentModalityDTO.java new file mode 100644 index 00000000..20d1053c --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/PaymentModalityDTO.java @@ -0,0 +1,16 @@ +package org.mifos.processor.bulk.schema; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class PaymentModalityDTO { + private String paymentModality; + private String financialAddress; + private String bankingInstitutionCode; +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeMessages.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeMessages.java index 71fa471f..a3e442cd 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeMessages.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeMessages.java @@ -5,5 +5,6 @@ public class ZeebeMessages { private ZeebeMessages() {} public static final String OPERATOR_MANUAL_RECOVERY = "operator-manual-recovery"; + public static final String ACCOUNT_LOOKUP = "account-lookup"; } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 9c0dd312..959fa45e 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -1,5 +1,7 @@ package org.mifos.processor.bulk.zeebe; +import com.amazonaws.services.dynamodbv2.xspec.S; + public class ZeebeVariables { private ZeebeVariables() { @@ -115,6 +117,12 @@ private ZeebeVariables() { public static final String PHASE_COUNT = "phaseCount"; + public static final String INITIATOR_FSP_ID = "initiatorFspId"; + public static final String ACCOUNT_LOOKUP_RETRY_COUNT = "accountLookupRetryCount"; + public static final String ACCOUNT_LOOKUP_FAILED = "accountLookupFailed"; + public static final String ORIGIN_CHANNEL_REQUEST = "originChannelRequest"; + public static final String CALLBACK = "X-CallbackURL"; + public static final String DEBULKINGDFSPID = "debulkingDfspid"; public static final String FILE_VALIDITY = "isFileValid"; diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java new file mode 100644 index 00000000..2b0ab047 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java @@ -0,0 +1,50 @@ +package org.mifos.processor.bulk.zeebe.worker; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.zeebe.client.ZeebeClient; +import org.apache.camel.CamelContext; +import org.mifos.connector.common.channel.dto.TransactionChannelRequestDTO; +import org.mifos.connector.common.mojaloop.dto.Party; +import org.mifos.connector.common.mojaloop.dto.PartyIdInfo; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.Map; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_PARTY_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CHANNEL_REQUEST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORIGIN_CHANNEL_REQUEST; +import static org.mifos.processor.bulk.zeebe.worker.Worker.ACCOUNT_LOOKUP_CALLBACK; + +@Component +public class AccountLookupCallbackWorker extends BaseWorker{ + @Autowired + private ZeebeClient zeebeClient; + @Autowired + private CamelContext camelContext; + @Autowired + private ObjectMapper objectMapper; + @Override + public void setup(){ + logger.info("## generating " + ACCOUNT_LOOKUP_CALLBACK + "zeebe worker"); + zeebeClient.newWorker() + .jobType(ACCOUNT_LOOKUP_CALLBACK.getValue()) + .handler((client, job) -> { + logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); + Map existingVariables = job.getVariablesAsMap(); + existingVariables.put(ORIGIN_CHANNEL_REQUEST,existingVariables.get(CHANNEL_REQUEST)); + TransactionChannelRequestDTO transactionChannelRequestDTO = objectMapper.readValue((String) existingVariables.get(CHANNEL_REQUEST), TransactionChannelRequestDTO.class); + String payeeId = existingVariables.get(PAYEE_PARTY_ID).toString(); + PartyIdInfo partyIdInfo = new PartyIdInfo(transactionChannelRequestDTO.getPayee().getPartyIdInfo().getPartyIdType(), payeeId); + Party payee =new Party(partyIdInfo); + transactionChannelRequestDTO.setPayee(payee); + existingVariables.put(CHANNEL_REQUEST, transactionChannelRequestDTO); + client.newCompleteCommand(job.getKey()) + .variables(existingVariables) + .send() + ; + }) + .name(String.valueOf(ACCOUNT_LOOKUP_CALLBACK)) + .open(); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java new file mode 100644 index 00000000..b0e12a5c --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java @@ -0,0 +1,85 @@ +package org.mifos.processor.bulk.zeebe.worker; +import com.amazonaws.services.dynamodbv2.xspec.S; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.zeebe.client.ZeebeClient; +import org.apache.camel.CamelContext; +import org.apache.camel.Exchange; +import org.apache.camel.ProducerTemplate; +import org.apache.camel.support.DefaultExchange; +import org.mifos.connector.common.channel.dto.TransactionChannelRequestDTO; +import org.mifos.connector.common.mojaloop.dto.Party; +import org.mifos.connector.common.mojaloop.dto.PartyIdInfo; +import org.mifos.connector.common.mojaloop.type.IdentifierType; +import org.mifos.processor.bulk.camel.config.CamelProperties; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.Map; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; +import static org.mifos.processor.bulk.zeebe.worker.Worker.ACCOUNT_LOOKUP; +import static org.mifos.processor.bulk.zeebe.worker.Worker.ACCOUNT_LOOKUP_CALLBACK; + +@Component +public class AccountLookupWorker extends BaseWorker{ + @Autowired + private ZeebeClient zeebeClient; + + @Autowired + private ProducerTemplate producerTemplate; + @Autowired + private CamelContext camelContext; + @Autowired + private ObjectMapper objectMapper; + @Value("${identity-account-mapper.hostname}") + private String identityMapperURL; + @Value("${bulk-processor.hostname}") + private String bulkURL; + @Override + public void setup() { + logger.info("## generating " + ACCOUNT_LOOKUP + "zeebe worker"); + zeebeClient.newWorker() + .jobType(ACCOUNT_LOOKUP.getValue()) + .handler((client, job) -> { + logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); + Map existingVariables = job.getVariablesAsMap(); + existingVariables.put(ACCOUNT_LOOKUP_RETRY_COUNT, 1); + existingVariables.put(CACHED_TRANSACTION_ID, job.getKey()); + + boolean isTransactionRequest = (boolean) existingVariables.get(IS_RTP_REQUEST); + String tenantId = (String) existingVariables.get(TENANT_ID); + Object channelRequest = existingVariables.get(CHANNEL_REQUEST); + TransactionChannelRequestDTO request = objectMapper.readValue((String) channelRequest, TransactionChannelRequestDTO.class); + + existingVariables.put(INITIATOR_FSP_ID, tenantId); + PartyIdInfo requestedParty = isTransactionRequest ? request.getPayer().getPartyIdInfo() : request.getPayee().getPartyIdInfo(); + + String payeeIdentity = requestedParty.getPartyIdentifier(); + String paymentModality = requestedParty.getPartyIdType().toString(); + + + Exchange exchange = new DefaultExchange(camelContext); + exchange.setProperty(HOST, identityMapperURL); + exchange.setProperty(PAYEE_IDENTITY, payeeIdentity); + exchange.setProperty(PAYMENT_MODALITY, paymentModality); + exchange.setProperty(CALLBACK,bulkURL + "/accountLookup/Callback"); + exchange.setProperty(TRANSACTION_ID, existingVariables.get(TRANSACTION_ID)); + exchange.setProperty("requestId", job.getKey()); + exchange.setProperty(CHANNEL_REQUEST, channelRequest); + exchange.setProperty(ORIGIN_DATE, existingVariables.get(ORIGIN_DATE)); + exchange.setProperty(TENANT_ID, tenantId); + producerTemplate.send("direct:send-account-lookup", exchange); + + + client.newCompleteCommand(job.getKey()) + .variables(existingVariables) + .send() + ; + }) + .name(String.valueOf(ACCOUNT_LOOKUP)) + .open(); + + } +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java index 0e4c79c2..c995e58a 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java @@ -11,7 +11,9 @@ public enum Worker { SEND_CALLBACK("sendCallback"), MERGE_BACK("mergeSubBatch"), - INIT_SUB_BATCH("initSubBatch"); + INIT_SUB_BATCH("initSubBatch"), + ACCOUNT_LOOKUP("accountLookup"), + ACCOUNT_LOOKUP_CALLBACK("accountLookupCallback"); private final String value; diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 300f3ef9..30781af7 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -29,7 +29,7 @@ zeebe: contactpoint: "zeebe-zeebe-gateway:26500" operations-app: - contactpoint: "https://ops-bk.sandbox.fynarfin.io" + contactpoint: "https://ops-bk.sandbox.mifos.io" username: "mifos" password: "password" endpoints: @@ -117,7 +117,12 @@ callback-phases: - 80 - 100 +identity-account-mapper: + hostname : "https://identity.sandbox.mifos.io" + +bulk-processor: + hostname : "https://bulk-connector.sandbox.mifos.io" csv: columnNames: "id,request_id,payment_mode,payer_identifier_type,payer_identifier,payee_identifier_type,payee_identifier,amount,currency,note" - size : 100000 # in bytes \ No newline at end of file + size : 100000 # in bytes From ff7de6977ff958827dfc54090f18823a6d078c8b Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Fri, 2 Jun 2023 11:49:58 +0530 Subject: [PATCH 072/156] FIX startup issue after the java 17 upgrade (#68) * Added auto dependency management for spring * Added camel-jackson dependency and organised dep order * Organised the dependency order * Removed auto dep management plugin and synced the dep based on plugin manually --- build.gradle | 56 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/build.gradle b/build.gradle index e60e790b..972f91b5 100644 --- a/build.gradle +++ b/build.gradle @@ -24,38 +24,56 @@ repositories { } } +ext { + camelVersion = '3.4.0' + springBootVersion = '2.6.2' + cucumberVersion = '7.8.1' + lambokVersion = '1.18.24' +} + dependencies { + // spring dependency + implementation "org.springframework.boot:spring-boot-starter:$springBootVersion" + implementation "org.springframework.boot:spring-boot-starter-web:$springBootVersion" + implementation "org.springframework:spring-web:5.3.14" + implementation 'org.springframework.kafka:spring-kafka:2.8.1' + + // spring test dependency + testImplementation "org.springframework.boot:spring-boot-starter-test:$springBootVersion" + + // camel dependency + implementation "org.apache.camel.springboot:camel-spring-boot-starter:$camelVersion" + implementation "org.apache.camel:camel-undertow:$camelVersion" + implementation "org.apache.camel:camel-http:$camelVersion" + implementation "org.apache.camel.springboot:camel-mail-starter:$camelVersion" + implementation "org.apache.camel:camel-jackson:$camelVersion" + + // camel test dependency + testImplementation "org.apache.camel:camel-test:$camelVersion" + testImplementation "org.apache.camel:camel-test-spring-junit5:$camelVersion" + + // cucumber test dependency + testImplementation "io.cucumber:cucumber-junit:$cucumberVersion" + testImplementation "io.cucumber:cucumber-spring:$cucumberVersion" + testImplementation "io.cucumber:cucumber-java:$cucumberVersion" + + // miscellaneous dependency implementation 'com.google.code.gson:gson:2.8.9' implementation 'org.json:json:20210307' implementation 'org.mifos:ph-ee-connector-common:1.4.1-SNAPSHOT' - implementation 'org.apache.camel.springboot:camel-spring-boot-starter:3.4.0' - implementation 'org.apache.camel:camel-undertow:3.4.0' - implementation 'org.springframework.boot:spring-boot-starter:2.6.2' - implementation 'org.springframework.boot:spring-boot-starter-web:2.6.2' - implementation 'org.springframework:spring-web:5.3.19' - implementation 'org.apache.camel:camel-http:3.4.0' implementation 'com.amazonaws:aws-java-sdk-s3:1.11.486' //To be removed implementation 'com.amazonaws:aws-java-sdk-dynamodb:1.11.486' - implementation 'com.azure:azure-storage-blob:12.12.0' implementation 'io.camunda:zeebe-client-java:8.1.1' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.12.3' - implementation 'org.springframework.kafka:spring-kafka:2.5.8.RELEASE' - - implementation 'org.apache.camel.springboot:camel-mail-starter:3.4.0' - implementation 'org.projectlombok:lombok:1.18.24' implementation 'org.apache.tika:tika-core:1.4' implementation 'org.apache.commons:commons-io:1.3.2' - annotationProcessor 'org.projectlombok:lombok:1.18.24' - - testImplementation 'org.apache.camel:camel-test-spring-junit5:3.4.0' - testImplementation 'io.cucumber:cucumber-junit:7.8.1' - testImplementation 'io.cucumber:cucumber-spring:7.8.1' - testImplementation 'io.cucumber:cucumber-java:7.8.1' - testImplementation 'org.apache.camel:camel-test:3.4.0' - testImplementation 'org.springframework.boot:spring-boot-starter-test:2.6.2' + implementation "org.projectlombok:lombok:$lambokVersion" + annotationProcessor "org.projectlombok:lombok:$lambokVersion" + + // miscellaneous test dependency testImplementation "com.google.truth:truth:1.1.3" testImplementation 'com.google.code.gson:gson:2.9.0' } From d2680a70a88fffa8568da54762553e58d9b9e30e Mon Sep 17 00:00:00 2001 From: Apurb Rajdhan <103931815+apurbraj@users.noreply.github.com> Date: Fri, 2 Jun 2023 18:37:34 +0530 Subject: [PATCH 073/156] Publishing Payee DfspId (#70) --- .../bulk/camel/processor/AccountLookupCallbackProcessor.java | 1 + .../java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java | 1 + .../bulk/zeebe/worker/AccountLookupCallbackWorker.java | 5 +++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/processor/AccountLookupCallbackProcessor.java b/src/main/java/org/mifos/processor/bulk/camel/processor/AccountLookupCallbackProcessor.java index 3b2f95be..d4d98564 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/processor/AccountLookupCallbackProcessor.java +++ b/src/main/java/org/mifos/processor/bulk/camel/processor/AccountLookupCallbackProcessor.java @@ -39,6 +39,7 @@ public void process(Exchange exchange) throws Exception { variables.put(ACCOUNT_LOOKUP_FAILED, false); variables.put(PAYEE_PARTY_ID, accountLookupResponseDTO.getPaymentModalityList().get(0).getFinancialAddress()); variables.put(PAYEE_PARTY_ID_TYPE, accountLookupResponseDTO.getPaymentModalityList().get(0).getPaymentModality()); + variables.put(PARTY_LOOKUP_FSP_ID, accountLookupResponseDTO.getPaymentModalityList().get(0).getBankingInstitutionCode()); exchange.setProperty(CACHED_TRANSACTION_ID, accountLookupResponseDTO.getRequestId()); }catch(IOException e){ variables.put(ACCOUNT_LOOKUP_FAILED, true); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 959fa45e..9324bf4b 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -128,5 +128,6 @@ private ZeebeVariables() { public static final String FILE_VALIDITY = "isFileValid"; public static final String NOTE = "note"; + public static final String PARTY_LOOKUP_FSP_ID = "partyLookupFspId"; } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java index 2b0ab047..b9db7e9e 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java @@ -38,13 +38,14 @@ public void setup(){ PartyIdInfo partyIdInfo = new PartyIdInfo(transactionChannelRequestDTO.getPayee().getPartyIdInfo().getPartyIdType(), payeeId); Party payee =new Party(partyIdInfo); transactionChannelRequestDTO.setPayee(payee); - existingVariables.put(CHANNEL_REQUEST, transactionChannelRequestDTO); + existingVariables.put(CHANNEL_REQUEST, objectMapper.writeValueAsString(transactionChannelRequestDTO)); client.newCompleteCommand(job.getKey()) .variables(existingVariables) .send() + .join() ; }) - .name(String.valueOf(ACCOUNT_LOOKUP_CALLBACK)) + .name(ACCOUNT_LOOKUP_CALLBACK.getValue()) .open(); } } From d0de2e38c7fb889423f5e10519b90b9a67469cc9 Mon Sep 17 00:00:00 2001 From: Arka Das <127482132+arkadasfynarfin@users.noreply.github.com> Date: Mon, 5 Jun 2023 16:30:41 +0530 Subject: [PATCH 074/156] Remove duplicate transactions from input CSV (#63) * remove duplicate transactions * enable ordering * camel route works when ordering is enabled * GOV-325: set note for duplicate transactions * GOV-325: perform deduplication when enabled * add null pointer checks * address review changes * address review comments --- .../bulk/camel/routes/OrderingRoute.java | 3 + .../bulk/zeebe/worker/OrderingWorker.java | 87 ++++++++++++++++--- .../bulk/zeebe/worker/WorkerConfig.java | 3 + src/main/resources/application.yaml | 6 +- 4 files changed, 85 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java index d5261416..b504b4cb 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java @@ -60,6 +60,9 @@ public void configure() { case "account_number": key = transaction.getAccountNumber(); break; + case "payee_identifier": + key = transaction.getPayeeIdentifier(); + break; case "amount": key = transaction.getAmount(); break; diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java index 4e8e932b..e4e7e74f 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java @@ -3,11 +3,17 @@ import org.apache.camel.Exchange; import org.apache.camel.support.DefaultExchange; import org.mifos.processor.bulk.camel.routes.RouteId; +import org.mifos.processor.bulk.schema.Transaction; import org.springframework.stereotype.Component; +import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.HashSet; +import java.util.Objects; import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @Component @@ -26,26 +32,83 @@ public void setup() { newWorker(Worker.ORDERING, (client, job) -> { logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); Map variables = job.getVariablesAsMap(); + Exchange exchange = new DefaultExchange(camelContext); + if (workerConfig.isOrderingWorkerEnabled) { variables.put(ORDERING_FAILED, false); + String filename = (String) variables.get(FILE_NAME); + exchange.setProperty(SERVER_FILE_NAME, filename); + + try { + sendToCamelRoute(RouteId.ORDERING, exchange); + assert !exchange.getProperty(ORDERING_FAILED, Boolean.class); + } catch (Exception e) { + variables.put(ORDERING_FAILED, true); + } + variables.put(ORDERING_FAILED, false); + variables.put(ORDERED_BY, exchange.getProperty(ORDERED_BY)); } - String filename = (String) variables.get(FILE_NAME); - Exchange exchange = new DefaultExchange(camelContext); - exchange.setProperty(SERVER_FILE_NAME, filename); + if(workerConfig.isTransactionDeduplicationEnabled){ + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + removeDuplicates(transactionList, workerConfig.isOrderingWorkerEnabled); + variables.put(TRANSACTION_LIST, transactionList); + } + client.newCompleteCommand(job.getKey()).variables(variables).send(); + }); + } + + private void removeDuplicates(List transactionList, boolean orderingEnabled){ + if(orderingEnabled){ + removeDuplicatesIfOrderingEnabled(transactionList); + return; + } + removeDuplicatesIfOrderingDisabled(transactionList); + } + + private void removeDuplicatesIfOrderingEnabled(List transactionList){ - try { - sendToCamelRoute(RouteId.ORDERING, exchange); - assert !exchange.getProperty(ORDERING_FAILED, Boolean.class); - } catch (Exception e) { - variables.put(ORDERING_FAILED, true); + for (int i = 0; i transactionList){ + Set set = new HashSet<>(); - client.newCompleteCommand(job.getKey()).variables(variables).send(); - }); + if(Objects.isNull(transactionList)){ + return; + } + + for(Transaction transaction : transactionList){ + String payeeDetail = fetchPayeeDetail(transaction); + if(set.contains(payeeDetail)){ + transaction.setNote("Duplicate transaction."); + } + else{ + set.add(payeeDetail); + } + } + } + + private String fetchPayeeDetail(Transaction transaction){ + String payeeIdentifier = transaction.getPayeeIdentifier(); + String payeeIdentifierType = transaction.getPayeeIdentifierType(); + String amount = transaction.getAmount(); + String currency = transaction.getCurrency(); + + return String.format("%s%s%s%s", payeeIdentifier, payeeIdentifierType, amount, currency); } } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java index 50ac4c58..a8eda20e 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java @@ -27,4 +27,7 @@ public class WorkerConfig { @Value("${config.completion-threshold-check.enable}") public boolean isCompletionThresholdCheckEnabled; + @Value("config.deduplication.enabled") + public boolean isTransactionDeduplicationEnabled; + } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 30781af7..ef56a688 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -84,8 +84,8 @@ config: approval: enable: true ordering: - enable: false - field: "payment_mode" + enable: true + field: "payee_identifier" splitting: enable: false sub-batch-size: 5 @@ -101,6 +101,8 @@ config: completion-threshold: 95 # in percentage max-retry: 4 #can be as high as 30 delay: 60 # in seconds + deduplication: + enabled: true callback: max-retry: 3 From b894895c9a0ea28e8682d64fc6a5512c09583011 Mon Sep 17 00:00:00 2001 From: Ishan Ranasingh <44061541+truthfool@users.noreply.github.com> Date: Wed, 14 Jun 2023 18:37:00 +0530 Subject: [PATCH 075/156] [GOV-261] Logging client IP (#71) * logging client ip * changed log level --- src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java b/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java index 159c0e36..aa1289aa 100644 --- a/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java +++ b/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java @@ -28,6 +28,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha HttpServletRequest req = (HttpServletRequest) request; String tenant = req.getHeader("Platform-TenantId"); logger.debug("Tenant Name is : {}", tenant); + logger.info("Client IP Address: {}",req.getRemoteHost()); } @Override From b537f7df8eb9c8f70609cbf42fcd05389f1bd209 Mon Sep 17 00:00:00 2001 From: Dhruv Sonagara <78945411+dhruvsonagara@users.noreply.github.com> Date: Wed, 14 Jun 2023 23:39:37 +0530 Subject: [PATCH 076/156] PHEE-307 Applied formatting and styling (#72) * Resolved conflicts * Applied formatting and styling Added licence file * Added license file --- .circleci/config.yml | 5 +- LICENSE.md | 373 ++++++++++++++++++ build.gradle | 159 +++++++- config/bulk-cleanup.xml | 71 ++++ config/bulk-formatter.xml | 366 +++++++++++++++++ config/checkstyle/checkstyle.xml | 266 +++++++++++++ config/checkstyle/suppressions.xml | 17 + kubernetes/phee-bulk-processor.yml | 2 +- .../processor/BulkProcessorApplication.java | 3 +- .../bulk/ConfigurationValidator.java | 27 +- .../org/mifos/processor/bulk/HealthCheck.java | 31 +- .../processor/bulk/OperationsAppConfig.java | 3 +- .../processor/bulk/api/ApiOriginFilter.java | 11 +- .../bulk/camel/config/CamelContextConfig.java | 8 +- .../bulk/camel/config/CamelProperties.java | 3 +- .../HttpClientConfigurerTrustAllCACerts.java | 22 +- .../AccountLookupCallbackProcessor.java | 35 +- .../processor/DTOJsonConversionException.java | 2 +- .../bulk/camel/processor/GsmaApiPayload.java | 5 +- .../camel/processor/MojaloopApiPayload.java | 5 +- .../bulk/camel/routes/AccountLookupRoute.java | 47 +-- .../bulk/camel/routes/BaseRouteBuilder.java | 24 +- .../bulk/camel/routes/BatchStatusRoute.java | 71 ++-- .../camel/routes/ExternalApiCallRoute.java | 28 +- .../camel/routes/FileProcessingRoute.java | 114 +++--- .../bulk/camel/routes/FileRoute.java | 61 ++- .../bulk/camel/routes/FormattingRoute.java | 64 ++- .../bulk/camel/routes/InitSubBatchRoute.java | 186 ++++----- .../bulk/camel/routes/MergeBackRoute.java | 75 ++-- .../camel/routes/OperationsAuthRoute.java | 58 ++- .../bulk/camel/routes/OrderingRoute.java | 123 +++--- .../camel/routes/ProcessorStartRoute.java | 339 ++++++++-------- .../processor/bulk/camel/routes/RouteId.java | 12 +- .../processor/bulk/camel/routes/Routes.java | 104 +++-- .../bulk/camel/routes/SendCallbackRoute.java | 83 ++-- .../bulk/camel/routes/SplittingRoute.java | 121 +++--- .../bulk/config/ExternalApiPayloadConfig.java | 12 +- .../bulk/config/PaymentModeConfiguration.java | 11 +- .../bulk/config/PaymentModeMapping.java | 4 +- .../bulk/config/PaymentModeType.java | 3 +- .../bulk/file/AwsFileTransferImpl.java | 8 +- .../bulk/file/AzureFileTransferImpl.java | 17 +- .../bulk/file/FileTransferService.java | 3 +- .../bulk/file/config/AwsStorageConfig.java | 8 +- .../bulk/file/config/AzureStorageConfig.java | 6 +- .../mifos/processor/bulk/format/Standard.java | 4 +- .../processor/bulk/format/StandardValue.java | 5 +- .../bulk/format/helper/BaseMapper.java | 3 +- .../bulk/format/helper/GSMAMapper.java | 1 + .../processor/bulk/format/helper/Mapper.java | 12 +- .../mifos/processor/bulk/kafka/Consumers.java | 54 ++- .../kafka/config/KafkaConsumerConfig.java | 5 +- .../kafka/config/KafkaProducerConfig.java | 5 +- .../bulk/kafka/config/KafkaTopicConfig.java | 6 +- .../bulk/schema/AccountLookupResponseDTO.java | 6 +- .../mifos/processor/bulk/schema/BatchDTO.java | 40 +- .../bulk/schema/GSMATransaction.java | 1 - .../bulk/schema/PaymentModalityDTO.java | 1 + .../processor/bulk/schema/Transaction.java | 3 +- .../bulk/schema/TransactionResult.java | 5 +- .../processor/bulk/utility/PhaseUtils.java | 5 +- .../mifos/processor/bulk/utility/Utils.java | 47 ++- .../bulk/zeebe/ZeebeClientConfiguration.java | 13 +- .../bulk/zeebe/ZeebeProcessStarter.java | 32 +- .../processor/bulk/zeebe/ZeebeVariables.java | 5 +- .../processor/bulk/zeebe/ZeebeWorkers.java | 139 +++---- .../worker/AccountLookupCallbackWorker.java | 52 ++- .../zeebe/worker/AccountLookupWorker.java | 92 +++-- .../bulk/zeebe/worker/ApprovalWorker.java | 4 +- .../bulk/zeebe/worker/BaseWorker.java | 10 +- .../bulk/zeebe/worker/BatchStatusWorker.java | 12 +- .../bulk/zeebe/worker/FormattingWorker.java | 18 +- .../bulk/zeebe/worker/InitSubBatchWorker.java | 30 +- .../bulk/zeebe/worker/MergeBackWorker.java | 20 +- .../bulk/zeebe/worker/OrderingWorker.java | 52 ++- .../bulk/zeebe/worker/PartyLookupWorker.java | 6 +- .../bulk/zeebe/worker/SendCallbackWorker.java | 48 ++- .../bulk/zeebe/worker/SplittingWorker.java | 31 +- .../processor/bulk/zeebe/worker/Worker.java | 14 +- .../BulkProcessorApplicationTests.java | 3 +- .../processor/cucumber/CucumberContext.java | 4 +- .../stepdef/ConfigurationTestStepDef.java | 8 +- .../cucumber/stepdef/InitRouteStepDef.java | 34 +- 83 files changed, 2414 insertions(+), 1412 deletions(-) create mode 100644 LICENSE.md create mode 100644 config/bulk-cleanup.xml create mode 100644 config/bulk-formatter.xml create mode 100644 config/checkstyle/checkstyle.xml create mode 100644 config/checkstyle/suppressions.xml diff --git a/.circleci/config.yml b/.circleci/config.yml index 14165c20..ed5fcf96 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,7 @@ jobs: # - slack/notify: # event: pass # template: basic_success_1 - - run: ./gradlew clean bootJar + - run: ./gradlew clean bootJar - aws-ecr/build-and-push-image: aws-access-key-id: AWS_ACCESS_KEY_ID aws-secret-access-key: AWS_SECRET_ACCESS_KEY @@ -33,7 +33,7 @@ jobs: repo: phee-bulk-processor repo-scan-on-push: true role-arn: arn:aws:iam::419830066942:role/CustomAdmin - tag: latest + tag: latest # - run: ./gradlew cucumberCli # run tests! Slack Success/Fail Notification Step #- run: ./gradlew test @@ -44,4 +44,3 @@ workflows: context: - AWS - slack - diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 00000000..a612ad98 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/build.gradle b/build.gradle index 972f91b5..6652a46e 100644 --- a/build.gradle +++ b/build.gradle @@ -6,14 +6,17 @@ plugins { id 'java' id 'maven-publish' id 'eclipse' + id 'checkstyle' id 'org.springframework.boot' version '2.6.2' + id 'com.diffplug.spotless' version '6.19.0' + id 'com.github.hierynomus.license' version '0.16.1' + id 'com.github.jk1.dependency-license-report' version '2.1' } repositories { // mavenLocal() maven { url = uri('https://repo.maven.apache.org/maven2') - } maven { url "https://fynarfin.jfrog.io/artifactory/fyn-libs-release-local" @@ -24,6 +27,53 @@ repositories { } } +apply plugin:'com.diffplug.spotless' +spotless { + format 'misc', { + target '**/*.md', '**/*.properties', '**/.gitignore', '**/.openapi-generator-ignore', '**/*.yml', '**/*.xml', '**/**.json', '**/*.sql' + targetExclude '**/build/**', '**/bin/**', '**/.settings/**', '**/.idea/**', '**/.gradle/**', '**/gradlew.bat', '**/licenses/**', '**/banner.txt', '.vscode/**' + indentWithSpaces(4) + endWithNewline() + trimTrailingWhitespace() + } + + groovyGradle { + target '*.gradle', '**/*.gradle' + targetExclude '**/build/**' + greclipse() + indentWithSpaces(4) + endWithNewline() + trimTrailingWhitespace() + } + lineEndings 'UNIX' +} +// Configuration for Gradle license plug-in +// https://github.com/hierynomus/license-gradle-plugin +license { + header rootProject.file("$rootDir/LICENSE.md") + excludes([ + "**/gradlew*", + "**/git.properties", + "**/*.html", + "**/templates/**", + "**/features/**", + "**/results/**", + "**/package-info.java", + "**/keystore.jks", + "**/legacy-docs/**", + "**/banner.txt", + "**/build.gradle.mustache", + "**/pom.mustache", + "**/avro/**/*.java", + "**/org/apache/fineract/client/**/*.java" + ]) + strictCheck true +} + +licenseReport { + outputDir = "$projectDir/licenses" +} + ext { camelVersion = '3.4.0' springBootVersion = '2.6.2' @@ -72,12 +122,119 @@ dependencies { implementation 'org.apache.commons:commons-io:1.3.2' implementation "org.projectlombok:lombok:$lambokVersion" annotationProcessor "org.projectlombok:lombok:$lambokVersion" + checkstyle 'com.puppycrawl.tools:checkstyle:10.9.3' + checkstyle 'com.github.sevntu-checkstyle:sevntu-checks:1.44.1' + implementation 'com.diffplug.gradle.spotless:spotless:2.4.1' // miscellaneous test dependency testImplementation "com.google.truth:truth:1.1.3" testImplementation 'com.google.code.gson:gson:2.9.0' } +configure(this) { + // NOTE: order matters! + apply plugin: 'java' + apply plugin: 'idea' + apply plugin: 'eclipse' + apply plugin: 'checkstyle' + apply plugin: 'com.diffplug.spotless' + apply plugin: 'com.github.hierynomus.license' + apply plugin: 'project-report' + apply plugin: 'com.github.jk1.dependency-license-report' + configurations { + implementation.setCanBeResolved(true) + api.setCanBeResolved(true) + } + tasks.withType(JavaCompile) { + options.compilerArgs += [ + "-Xlint:unchecked", + "-Xlint:cast", + "-Xlint:auxiliaryclass", + "-Xlint:deprecation", + "-Xlint:dep-ann", + "-Xlint:divzero", + "-Xlint:empty", + "-Xlint:exports", + "-Xlint:fallthrough", + "-Xlint:finally", + "-Xlint:module", + "-Xlint:opens", + "-Xlint:options", + "-Xlint:overloads", + "-Xlint:overrides", + "-Xlint:path", + "-Xlint:processing", + "-Xlint:removal", + "-Xlint:requires-automatic", + "-Xlint:requires-transitive-automatic", + "-Xlint:try", + "-Xlint:varargs", + "-Xlint:preview", + "-Xlint:static", + // -Werror needs to be disabled because EclipseLink's static weaving doesn't generate warning-free code + // and during an IntelliJ recompilation, it fails + //"-Werror", + "-Xmaxwarns", + 1500, + "-Xmaxerrs", + 1500 + ] + options.deprecation = true + } + // Configuration for the spotless plugin + // https://github.com/diffplug/spotless/tree/main/plugin-gradle + spotless { + java { + targetExclude '**/build/**', '**/bin/**', '**/out/**' + importOrder() //sort imports alphabetically + removeUnusedImports() + eclipse().configFile "$rootDir/config/bulk-formatter.xml" + endWithNewline() + trimTrailingWhitespace() + // Enforce style modifier order + custom 'Modifier ordering', { + def modifierRanking = [ + public : 1, + protected : 2, + private : 3, + abstract : 4, + default : 5, + static : 6, + final : 7, + transient : 8, + volatile : 9, + synchronized: 10, + native : 11, + strictfp : 12] + // Find any instance of multiple modifiers. Lead with a non-word character to avoid + // accidental matching against for instance, "an alternative default value" + it.replaceAll(/\W(?:public |protected |private |abstract |default |static |final |transient |volatile |synchronized |native |strictfp ){2,}/, { + // Do not replace the leading non-word character. Identify the modifiers + it.replaceAll(/(?:public |protected |private |abstract |default |static |final |transient |volatile |synchronized |native |strictfp ){2,}/, { + // Sort the modifiers according to the ranking above + it.split().sort({ modifierRanking[it] }).join(' ') + ' ' + } + ) + } + ) + } + } + lineEndings 'UNIX' + } + // If we are running Gradle within Eclipse to enhance classes, + // set the classes directory to point to Eclipse's default build directory + if (project.hasProperty('env') && project.getProperty('env') == 'eclipse') { + sourceSets.main.java.outputDir = file("$projectDir/bin/main") + } + // Configuration for the Checkstyle plugin + // https://docs.gradle.org/current/userguide/checkstyle_plugin.html + dependencies { + checkstyle 'com.puppycrawl.tools:checkstyle:10.3.1' + checkstyle 'com.github.sevntu-checkstyle:sevntu-checks:1.42.0' + } +} + + group = 'org.mifos' version = '0.0.1-SNAPSHOT' description = 'ph-ee-processor-bulk' diff --git a/config/bulk-cleanup.xml b/config/bulk-cleanup.xml new file mode 100644 index 00000000..67cb2b70 --- /dev/null +++ b/config/bulk-cleanup.xml @@ -0,0 +1,71 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/config/bulk-formatter.xml b/config/bulk-formatter.xml new file mode 100644 index 00000000..21a66024 --- /dev/null +++ b/config/bulk-formatter.xml @@ -0,0 +1,366 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml new file mode 100644 index 00000000..5183efef --- /dev/null +++ b/config/checkstyle/checkstyle.xml @@ -0,0 +1,266 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/config/checkstyle/suppressions.xml b/config/checkstyle/suppressions.xml new file mode 100644 index 00000000..f4e5b54c --- /dev/null +++ b/config/checkstyle/suppressions.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + diff --git a/kubernetes/phee-bulk-processor.yml b/kubernetes/phee-bulk-processor.yml index 88816cdd..d9f4683a 100644 --- a/kubernetes/phee-bulk-processor.yml +++ b/kubernetes/phee-bulk-processor.yml @@ -44,4 +44,4 @@ spec: selector: app: ph-ee-bulk-processor sessionAffinity: None - type: LoadBalancer \ No newline at end of file + type: LoadBalancer diff --git a/src/main/java/org/mifos/processor/BulkProcessorApplication.java b/src/main/java/org/mifos/processor/BulkProcessorApplication.java index 280a85f1..4602e694 100644 --- a/src/main/java/org/mifos/processor/BulkProcessorApplication.java +++ b/src/main/java/org/mifos/processor/BulkProcessorApplication.java @@ -24,8 +24,7 @@ public ObjectMapper objectMapper() { ObjectMapper objectMapper = new ObjectMapper(); objectMapper.registerModule(new JavaTimeModule()); objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); - return objectMapper - .setSerializationInclusion(JsonInclude.Include.NON_NULL) + return objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL) .configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true) .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); } diff --git a/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java b/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java index 78a98a6e..826f6133 100644 --- a/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java +++ b/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java @@ -1,17 +1,17 @@ package org.mifos.processor.bulk; -import org.mifos.processor.bulk.schema.Transaction; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import javax.annotation.PostConstruct; import org.mifos.processor.bulk.format.Standard; +import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.zeebe.worker.WorkerConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.List; @Component public class ConfigurationValidator { @@ -65,24 +65,24 @@ private void validateFormattingStandard() { e.printStackTrace(); } List possibleStandards = new ArrayList<>(); - for (Field f: Standard.class.getFields()) { + for (Field f : Standard.class.getFields()) { possibleStandards.add(f.getName()); } - throw new ConfigurationValidationException("Invalid standard configured for formatting data. Possible values are [" - + String.join(",", possibleStandards) + "]"); + throw new ConfigurationValidationException( + "Invalid standard configured for formatting data. Possible values are [" + String.join(",", possibleStandards) + "]"); } // validates the ordering configuration private void validateOrderingConfig() { List possibleOrderingFields = new ArrayList<>(); - for (Field field: Transaction.class.getDeclaredFields()) { + for (Field field : Transaction.class.getDeclaredFields()) { possibleOrderingFields.add(field.getName()); } if (!possibleOrderingFields.contains(orderingField)) { - throw new ConfigurationValidationException("Invalid ordering field, possible values are [" - + String.join(",", possibleOrderingFields) + "]"); + throw new ConfigurationValidationException( + "Invalid ordering field, possible values are [" + String.join(",", possibleOrderingFields) + "]"); } } @@ -99,6 +99,9 @@ private void validateCompletionThresholdConfig() { // this exception is thrown when unexpected application config is set, and can't pass the ConfigurationValidator public static class ConfigurationValidationException extends RuntimeException { - ConfigurationValidationException(String message) { super(message); } + + ConfigurationValidationException(String message) { + super(message); + } } } diff --git a/src/main/java/org/mifos/processor/bulk/HealthCheck.java b/src/main/java/org/mifos/processor/bulk/HealthCheck.java index ef0edd9e..37d91a02 100644 --- a/src/main/java/org/mifos/processor/bulk/HealthCheck.java +++ b/src/main/java/org/mifos/processor/bulk/HealthCheck.java @@ -1,9 +1,12 @@ package org.mifos.processor.bulk; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; + import com.fasterxml.jackson.databind.MappingIterator; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import java.util.UUID; import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; import org.apache.camel.builder.RouteBuilder; @@ -17,10 +20,6 @@ import org.springframework.kafka.core.KafkaTemplate; import org.springframework.stereotype.Component; -import java.util.UUID; - -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; - @Component public class HealthCheck extends RouteBuilder { @@ -53,14 +52,10 @@ public class HealthCheck extends RouteBuilder { @Override public void configure() { - from("rest:GET:/") - .setHeader(Exchange.HTTP_RESPONSE_CODE, constant(200)) - .setBody(constant("")); - - from("rest:GET:/channel/bulk/transfer/{fileName}") - .id("transfer-details") - .log(LoggingLevel.INFO, "## CHANNEL -> inbound bulk transfer request with ${header.fileName}") - .process(exchange -> { + from("rest:GET:/").setHeader(Exchange.HTTP_RESPONSE_CODE, constant(200)).setBody(constant("")); + + from("rest:GET:/channel/bulk/transfer/{fileName}").id("transfer-details") + .log(LoggingLevel.INFO, "## CHANNEL -> inbound bulk transfer request with ${header.fileName}").process(exchange -> { String fileName = exchange.getIn().getHeader("fileName", String.class); String batchId = UUID.randomUUID().toString(); exchange.setProperty(BATCH_ID, batchId); @@ -69,19 +64,19 @@ public void configure() { byte[] csvFile = fileTransferService.downloadFile(fileName, bucketName); CsvSchema schema = CsvSchema.emptySchema().withHeader(); - MappingIterator readValues = csvMapper.readerWithSchemaFor(TransactionOlder.class).with(schema).readValues(csvFile); + MappingIterator readValues = csvMapper.readerWithSchemaFor(TransactionOlder.class).with(schema) + .readValues(csvFile); while (readValues.hasNext()) { TransactionOlder current = readValues.next(); current.setBatchId(batchId); logger.info("Writing string in kafka {}", objectMapper.writeValueAsString(current)); - if (current.getPaymentMode().equals("gsma") || current.getPaymentMode().equals("afrimoney")) + if (current.getPaymentMode().equals("gsma") || current.getPaymentMode().equals("afrimoney")) { kafkaTemplate.send(gsmaTopicName, objectMapper.writeValueAsString(current)); - else if (current.getPaymentMode().equals("sclb")) + } else if (current.getPaymentMode().equals("sclb")) { kafkaTemplate.send(slcbTopicName, objectMapper.writeValueAsString(current)); + } } - }) - .setHeader(Exchange.HTTP_RESPONSE_CODE, constant(200)) - .setBody(exchange -> exchange.getProperty(BATCH_ID)); + }).setHeader(Exchange.HTTP_RESPONSE_CODE, constant(200)).setBody(exchange -> exchange.getProperty(BATCH_ID)); } } diff --git a/src/main/java/org/mifos/processor/bulk/OperationsAppConfig.java b/src/main/java/org/mifos/processor/bulk/OperationsAppConfig.java index c738fba6..78b52478 100644 --- a/src/main/java/org/mifos/processor/bulk/OperationsAppConfig.java +++ b/src/main/java/org/mifos/processor/bulk/OperationsAppConfig.java @@ -1,10 +1,9 @@ package org.mifos.processor.bulk; +import javax.annotation.PostConstruct; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; - @Component public class OperationsAppConfig { diff --git a/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java b/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java index aa1289aa..46d29187 100644 --- a/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java +++ b/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java @@ -1,10 +1,6 @@ package org.mifos.processor.bulk.api; -import com.google.common.base.Optional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - +import java.io.IOException; import javax.servlet.Filter; import javax.servlet.FilterChain; import javax.servlet.FilterConfig; @@ -12,10 +8,13 @@ import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletRequest; -import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; @Component public class ApiOriginFilter implements Filter { + private Logger logger = LoggerFactory.getLogger(this.getClass()); @Override diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelContextConfig.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelContextConfig.java index a7a88538..6366dd11 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelContextConfig.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelContextConfig.java @@ -1,19 +1,15 @@ package org.mifos.processor.bulk.camel.config; +import java.util.HashMap; import org.apache.camel.CamelContext; import org.apache.camel.component.http.HttpComponent; import org.apache.camel.spi.RestConfiguration; import org.apache.camel.spring.boot.CamelContextConfiguration; -import org.apache.camel.support.jsse.SSLContextParameters; -import org.apache.camel.support.jsse.TrustManagersParameters; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import javax.net.ssl.X509ExtendedTrustManager; -import java.util.HashMap; - @Configuration public class CamelContextConfig { @@ -29,6 +25,7 @@ public class CamelContextConfig { @Bean CamelContextConfiguration contextConfiguration() { return new CamelContextConfiguration() { + @Override public void beforeApplicationStart(CamelContext camelContext) { camelContext.setTracing(false); @@ -41,7 +38,6 @@ public void beforeApplicationStart(CamelContext camelContext) { httpComponent.setHttpClientConfigurer(httpClientConfigurerTrustAllCACerts); } - RestConfiguration rest = new RestConfiguration(); camelContext.setRestConfiguration(rest); rest.setComponent("undertow"); diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 3977125a..292e15bc 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -5,7 +5,8 @@ public class CamelProperties { private CamelProperties() {} public static final String AUTH_TYPE = "authType"; - public static final String IS_BATCH_READY = "isBatchReady"; // camel property to check if batch is ready for sampling + public static final String IS_BATCH_READY = "isBatchReady"; // camel property to check if batch is ready for + // sampling public static final String SERVER_FILE_NAME = "serverFileName"; diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java b/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java index d87fd5bb..55efee86 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java @@ -13,10 +13,10 @@ import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.conn.ssl.TrustStrategy; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.ssl.SSLContextBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,8 +24,7 @@ public class HttpClientConfigurerTrustAllCACerts implements HttpClientConfigurer public Logger logger = LoggerFactory.getLogger(this.getClass()); - public HttpClientConfigurerTrustAllCACerts() { - } + public HttpClientConfigurerTrustAllCACerts() {} @Override public void configureHttpClient(HttpClientBuilder clientBuilder) { @@ -34,6 +33,7 @@ public void configureHttpClient(HttpClientBuilder clientBuilder) { SSLContext sslContext = null; try { sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() { + public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException { return true; } @@ -41,25 +41,23 @@ public boolean isTrusted(X509Certificate[] arg0, String arg1) throws Certificate } catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) { e.printStackTrace(); } - clientBuilder.setSslcontext( sslContext); + clientBuilder.setSslcontext(sslContext); // don't check Hostnames, either. - // -- use SSLConnectionSocketFactory.getDefaultHostnameVerifier(), if you don't want to weaken + // -- use SSLConnectionSocketFactory.getDefaultHostnameVerifier(), if you don't want to weaken HostnameVerifier hostnameVerifier = SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER; // here's the special part: - // -- need to create an SSL Socket Factory, to use our weakened "trust strategy"; - // -- and create a Registry, to register it. + // -- need to create an SSL Socket Factory, to use our weakened "trust strategy"; + // -- and create a Registry, to register it. // SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, hostnameVerifier); Registry socketFactoryRegistry = RegistryBuilder.create() - .register("http", PlainConnectionSocketFactory.getSocketFactory()) - .register("https", sslSocketFactory) - .build(); + .register("http", PlainConnectionSocketFactory.getSocketFactory()).register("https", sslSocketFactory).build(); // now, we create connection-manager using our Registry. - // -- allows multi-threaded use - PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager( socketFactoryRegistry); + // -- allows multi-threaded use + PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager(socketFactoryRegistry); clientBuilder.setConnectionManager(connMgr); } diff --git a/src/main/java/org/mifos/processor/bulk/camel/processor/AccountLookupCallbackProcessor.java b/src/main/java/org/mifos/processor/bulk/camel/processor/AccountLookupCallbackProcessor.java index d4d98564..3065e73a 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/processor/AccountLookupCallbackProcessor.java +++ b/src/main/java/org/mifos/processor/bulk/camel/processor/AccountLookupCallbackProcessor.java @@ -1,7 +1,18 @@ package org.mifos.processor.bulk.camel.processor; +import static org.mifos.processor.bulk.camel.config.CamelProperties.CACHED_TRANSACTION_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_PARTY_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_PARTY_ID_TYPE; +import static org.mifos.processor.bulk.zeebe.ZeebeMessages.ACCOUNT_LOOKUP; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ACCOUNT_LOOKUP_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_FSP_ID; + import com.fasterxml.jackson.databind.ObjectMapper; import io.camunda.zeebe.client.ZeebeClient; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; import org.apache.camel.Exchange; import org.apache.camel.Processor; import org.mifos.processor.bulk.schema.AccountLookupResponseDTO; @@ -10,23 +21,16 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.io.IOException; -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; - -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; -import static org.mifos.processor.bulk.zeebe.ZeebeMessages.ACCOUNT_LOOKUP; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; - @Component public class AccountLookupCallbackProcessor implements Processor { + private final Logger logger = LoggerFactory.getLogger(this.getClass()); @Autowired(required = false) private ZeebeClient zeebeClient; @Autowired private ObjectMapper objectMapper; + @Override public void process(Exchange exchange) throws Exception { Map variables = new HashMap<>(); @@ -41,19 +45,16 @@ public void process(Exchange exchange) throws Exception { variables.put(PAYEE_PARTY_ID_TYPE, accountLookupResponseDTO.getPaymentModalityList().get(0).getPaymentModality()); variables.put(PARTY_LOOKUP_FSP_ID, accountLookupResponseDTO.getPaymentModalityList().get(0).getBankingInstitutionCode()); exchange.setProperty(CACHED_TRANSACTION_ID, accountLookupResponseDTO.getRequestId()); - }catch(IOException e){ + } catch (IOException e) { variables.put(ACCOUNT_LOOKUP_FAILED, true); error = objectMapper.readValue(response, String.class); } - if(zeebeClient != null) { + if (zeebeClient != null) { - zeebeClient.newPublishMessageCommand() - .messageName(ACCOUNT_LOOKUP) - .correlationKey(exchange.getProperty(CACHED_TRANSACTION_ID, String.class)) - .timeToLive(Duration.ofMillis(50000)) - .variables(variables) - .send(); + zeebeClient.newPublishMessageCommand().messageName(ACCOUNT_LOOKUP) + .correlationKey(exchange.getProperty(CACHED_TRANSACTION_ID, String.class)).timeToLive(Duration.ofMillis(50000)) + .variables(variables).send(); } } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/processor/DTOJsonConversionException.java b/src/main/java/org/mifos/processor/bulk/camel/processor/DTOJsonConversionException.java index 954325be..5f59e693 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/processor/DTOJsonConversionException.java +++ b/src/main/java/org/mifos/processor/bulk/camel/processor/DTOJsonConversionException.java @@ -1,6 +1,6 @@ package org.mifos.processor.bulk.camel.processor; -public class DTOJsonConversionException extends RuntimeException { +public class DTOJsonConversionException extends RuntimeException { public DTOJsonConversionException(Class dtoClass) { super("Unable to convert " + dtoClass.getName() + "to json string"); diff --git a/src/main/java/org/mifos/processor/bulk/camel/processor/GsmaApiPayload.java b/src/main/java/org/mifos/processor/bulk/camel/processor/GsmaApiPayload.java index 2bc4b699..17c8a17f 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/processor/GsmaApiPayload.java +++ b/src/main/java/org/mifos/processor/bulk/camel/processor/GsmaApiPayload.java @@ -1,15 +1,16 @@ package org.mifos.processor.bulk.camel.processor; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_ELEMENT; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.function.Function; import org.apache.camel.Exchange; import org.mifos.connector.common.gsma.dto.GSMATransaction; import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.utility.Utils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.util.function.Function; -import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_ELEMENT; @Component public class GsmaApiPayload implements Function { diff --git a/src/main/java/org/mifos/processor/bulk/camel/processor/MojaloopApiPayload.java b/src/main/java/org/mifos/processor/bulk/camel/processor/MojaloopApiPayload.java index 94ff7c73..b52aee39 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/processor/MojaloopApiPayload.java +++ b/src/main/java/org/mifos/processor/bulk/camel/processor/MojaloopApiPayload.java @@ -1,15 +1,16 @@ package org.mifos.processor.bulk.camel.processor; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_ELEMENT; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.function.Function; import org.apache.camel.Exchange; import org.mifos.connector.common.channel.dto.TransactionChannelRequestDTO; import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.utility.Utils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.util.function.Function; -import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_ELEMENT; @Component public class MojaloopApiPayload implements Function { diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java index 19edbdd8..7a3b83fd 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java @@ -1,46 +1,39 @@ package org.mifos.processor.bulk.camel.routes; +import static org.mifos.processor.bulk.camel.config.CamelProperties.ENDPOINT; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HOST; +import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_IDENTITY; +import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYMENT_MODALITY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; + import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; -import org.mifos.processor.bulk.camel.config.CamelProperties; import org.mifos.processor.bulk.camel.processor.AccountLookupCallbackProcessor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; - @Component -public class AccountLookupRoute extends BaseRouteBuilder{ +public class AccountLookupRoute extends BaseRouteBuilder { + @Autowired AccountLookupCallbackProcessor accountLookupCallbackProcessor; + @Override public void configure() throws Exception { - from("rest:POST:/accountLookup/Callback") - .log(LoggingLevel.DEBUG, "######## -> ACCOUNT LOOKUP CALLBACK") - .process(accountLookupCallbackProcessor) - .setBody(constant("Received")) - .setHeader(Exchange.HTTP_RESPONSE_CODE, constant(200)) + from("rest:POST:/accountLookup/Callback").log(LoggingLevel.DEBUG, "######## -> ACCOUNT LOOKUP CALLBACK") + .process(accountLookupCallbackProcessor).setBody(constant("Received")).setHeader(Exchange.HTTP_RESPONSE_CODE, constant(200)) .end(); - from("direct:send-account-lookup") - .id("account-lookup") - .process(exchange -> { - String callbackUrl = exchange.getProperty(CALLBACK, String.class); - exchange.getIn().setHeader(CALLBACK, callbackUrl); - }) - .setHeader(Exchange.HTTP_METHOD, constant("GET")) - .setHeader( - Exchange.HTTP_QUERY, - simple(new StringBuilder().append(PAYEE_IDENTITY).append("=${exchangeProperty.") - .append(PAYEE_IDENTITY).append("}&").append(PAYMENT_MODALITY) - .append("=${exchangeProperty.").append(PAYMENT_MODALITY) - .append("}&").append("requestId=${exchangeProperty.requestId}").toString() - ) - ) + from("direct:send-account-lookup").id("account-lookup").process(exchange -> { + String callbackUrl = exchange.getProperty(CALLBACK, String.class); + exchange.getIn().setHeader(CALLBACK, callbackUrl); + }).setHeader(Exchange.HTTP_METHOD, constant("GET")) + .setHeader(Exchange.HTTP_QUERY, + simple(new StringBuilder().append(PAYEE_IDENTITY).append("=${exchangeProperty.").append(PAYEE_IDENTITY).append("}&") + .append(PAYMENT_MODALITY).append("=${exchangeProperty.").append(PAYMENT_MODALITY).append("}&") + .append("requestId=${exchangeProperty.requestId}").toString())) .setProperty(HOST, simple("{{identity-account-mapper.hostname}}")) - .setProperty(ENDPOINT, simple("identityAccountMapper/accountLookup/")) - .to("direct:external-api-calling"); + .setProperty(ENDPOINT, simple("identityAccountMapper/accountLookup/")).to("direct:external-api-calling"); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java index 63c8ed29..9871445a 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java @@ -2,6 +2,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.camunda.zeebe.client.ZeebeClient; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; import org.apache.camel.Exchange; import org.apache.camel.builder.RouteBuilder; import org.apache.camel.model.RouteDefinition; @@ -10,10 +14,6 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.List; public abstract class BaseRouteBuilder extends RouteBuilder { @@ -35,22 +35,16 @@ public abstract class BaseRouteBuilder extends RouteBuilder { public Logger logger = LoggerFactory.getLogger(this.getClass()); public RouteDefinition getBaseExternalApiRequestRouteDefinition(String routeId, HttpRequestMethod httpMethod) { - return from(String.format("direct:%s", routeId)) - .id(routeId) - .log("Starting external API request route: " + routeId) - .removeHeader("*") - .setHeader(Exchange.HTTP_METHOD, constant(httpMethod.text)) - .setHeader("X-Date", simple(ZonedDateTime.now( ZoneOffset.UTC ).format( DateTimeFormatter.ISO_INSTANT ))) + return from(String.format("direct:%s", routeId)).id(routeId).log("Starting external API request route: " + routeId) + .removeHeader("*").setHeader(Exchange.HTTP_METHOD, constant(httpMethod.text)) + .setHeader("X-Date", simple(ZonedDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT))) .setHeader("Content-Type", constant("application/json;charset=UTF-8")) .setHeader("Accept", constant("application/json, text/plain, */*")); } protected enum HttpRequestMethod { - GET("GET"), - POST("POST"), - PUT("PUT"), - DELETE("DELETE") - ; + + GET("GET"), POST("POST"), PUT("PUT"), DELETE("DELETE"); private final String text; diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java index 05a4682e..8e2edeea 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java @@ -1,14 +1,19 @@ package org.mifos.processor.bulk.camel.routes; +import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_STATUS_FAILED; +import static org.mifos.processor.bulk.camel.config.CamelProperties.OPS_APP_ACCESS_TOKEN; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_RATE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_CODE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_DESCRIPTION; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; + import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; import org.apache.camel.model.dataformat.JsonLibrary; import org.mifos.processor.bulk.schema.BatchDTO; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_STATUS_FAILED; -import static org.mifos.processor.bulk.camel.config.CamelProperties.OPS_APP_ACCESS_TOKEN; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @Component public class BatchStatusRoute extends BaseRouteBuilder { @@ -19,65 +24,43 @@ public class BatchStatusRoute extends BaseRouteBuilder { @Override public void configure() throws Exception { - from("rest:get:test/batch/summary") - .to(RouteId.BATCH_STATUS.getValue()); + from("rest:get:test/batch/summary").to(RouteId.BATCH_STATUS.getValue()); /** - * Base route for kicking off ordering logic. Performs below tasks. - * 1. Downloads the csv form cloud. - * 2. Builds the [Transaction] array using [direct:get-transaction-array] route. - * 3. Format the data based on the configuration provided in application.yaml. @see [Standard.java] - * 4. Update file with the updated data. - * 5. Uploads the updated file in cloud. + * Base route for kicking off ordering logic. Performs below tasks. 1. Downloads the csv form cloud. 2. Builds + * the [Transaction] array using [direct:get-transaction-array] route. 3. Format the data based on the + * configuration provided in application.yaml. @see [Standard.java] 4. Update file with the updated data. 5. + * Uploads the updated file in cloud. */ - from(RouteId.BATCH_STATUS.getValue()) - .id(RouteId.BATCH_STATUS.getValue()) - .log("Starting route " + RouteId.BATCH_STATUS.name()) - .to("direct:get-access-token") - .choice() - .when(exchange -> exchange.getProperty(OPS_APP_ACCESS_TOKEN, String.class) != null) - .log(LoggingLevel.INFO, "Got access token, moving on to API call") - .to("direct:batch-summary") - .to("direct:batch-summary-response-handler") - .otherwise() - .log(LoggingLevel.INFO, "Authentication failed.") - .endChoice(); - + from(RouteId.BATCH_STATUS.getValue()).id(RouteId.BATCH_STATUS.getValue()).log("Starting route " + RouteId.BATCH_STATUS.name()) + .to("direct:get-access-token").choice().when(exchange -> exchange.getProperty(OPS_APP_ACCESS_TOKEN, String.class) != null) + .log(LoggingLevel.INFO, "Got access token, moving on to API call").to("direct:batch-summary") + .to("direct:batch-summary-response-handler").otherwise().log(LoggingLevel.INFO, "Authentication failed.").endChoice(); getBaseExternalApiRequestRouteDefinition("batch-summary", HttpRequestMethod.GET) .setHeader(Exchange.REST_HTTP_QUERY, simple("batchId=${exchangeProperty." + BATCH_ID + "}")) - .setHeader("Authorization", simple("Bearer ${exchangeProperty."+OPS_APP_ACCESS_TOKEN+"}")) - .setHeader("Platform-TenantId", simple("${exchangeProperty." + TENANT_ID + "}")) - .process(exchange -> { + .setHeader("Authorization", simple("Bearer ${exchangeProperty." + OPS_APP_ACCESS_TOKEN + "}")) + .setHeader("Platform-TenantId", simple("${exchangeProperty." + TENANT_ID + "}")).process(exchange -> { logger.info(exchange.getIn().getHeaders().toString()); - }) - .toD(operationsAppConfig.batchSummaryUrl + "?bridgeEndpoint=true") + }).toD(operationsAppConfig.batchSummaryUrl + "?bridgeEndpoint=true") .log(LoggingLevel.INFO, "Batch summary API response: \n\n ${body}"); - from("direct:batch-summary-response-handler") - .id("direct:batch-summary-response-handler") + from("direct:batch-summary-response-handler").id("direct:batch-summary-response-handler") .log("Starting route direct:batch-summary-response-handler") - //.setBody(exchange -> exchange.getIn().getBody(String.class)) - .choice() - .when(header("CamelHttpResponseCode").isEqualTo("200")) - .log(LoggingLevel.INFO, "Batch summary request successful") - .unmarshal().json(JsonLibrary.Jackson, BatchDTO.class) - .process(exchange -> { + // .setBody(exchange -> exchange.getIn().getBody(String.class)) + .choice().when(header("CamelHttpResponseCode").isEqualTo("200")).log(LoggingLevel.INFO, "Batch summary request successful") + .unmarshal().json(JsonLibrary.Jackson, BatchDTO.class).process(exchange -> { BatchDTO batchSummary = exchange.getIn().getBody(BatchDTO.class); - long percentage =(long)(((double)batchSummary.getSuccessful()/batchSummary.getTotal())*100); + long percentage = (long) (((double) batchSummary.getSuccessful() / batchSummary.getTotal()) * 100); if (percentage >= completionThreshold) { - logger.info("Batch success threshold reached. Expected rate: {}, Actual Rate: {}", - completionThreshold, percentage); + logger.info("Batch success threshold reached. Expected rate: {}, Actual Rate: {}", completionThreshold, percentage); } exchange.setProperty(COMPLETION_RATE, percentage); - }) - .otherwise() - .log(LoggingLevel.ERROR, "Batch summary request unsuccessful") - .process(exchange -> { + }).otherwise().log(LoggingLevel.ERROR, "Batch summary request unsuccessful").process(exchange -> { exchange.setProperty(BATCH_STATUS_FAILED, true); exchange.setProperty(ERROR_DESCRIPTION, exchange.getIn().getBody(String.class)); exchange.setProperty(ERROR_CODE, exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE)); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java index 8f549ed8..8035b40c 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java @@ -1,30 +1,28 @@ package org.mifos.processor.bulk.camel.routes; +import static org.mifos.processor.bulk.camel.config.CamelProperties.ENDPOINT; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HOST; + import org.apache.camel.LoggingLevel; import org.apache.camel.builder.RouteBuilder; import org.springframework.stereotype.Component; -import static org.mifos.processor.bulk.camel.config.CamelProperties.ENDPOINT; -import static org.mifos.processor.bulk.camel.config.CamelProperties.HOST; - @Component public class ExternalApiCallRoute extends RouteBuilder { + @Override public void configure() { - from("direct:external-api-calling") - .id("external-api-call") - .log(LoggingLevel.DEBUG,"######## API CALL -> Calling an external api") + from("direct:external-api-calling").id("external-api-call").log(LoggingLevel.DEBUG, "######## API CALL -> Calling an external api") .process(exchange -> { // remove the trailing "/" from endpoint String endpoint = exchange.getProperty(ENDPOINT, String.class); - if (endpoint.startsWith("/")) { exchange.setProperty(ENDPOINT, endpoint.substring(1)); } - }) - .log(LoggingLevel.DEBUG,"Host: ${exchangeProperty." + HOST + "}") - .log(LoggingLevel.DEBUG,"Endpoint: ${exchangeProperty." + ENDPOINT + "}") - .log(LoggingLevel.DEBUG,"Headers: ${headers}") - .log(LoggingLevel.DEBUG,"Request Body: ${body}") - .toD("${exchangeProperty." + HOST + "}/${exchangeProperty." + ENDPOINT + "}" + - "?bridgeEndpoint=true" + "&throwExceptionOnFailure=false") - .log(LoggingLevel.DEBUG,"Response body: ${body}"); + if (endpoint.startsWith("/")) { + exchange.setProperty(ENDPOINT, endpoint.substring(1)); + } + }).log(LoggingLevel.DEBUG, "Host: ${exchangeProperty." + HOST + "}") + .log(LoggingLevel.DEBUG, "Endpoint: ${exchangeProperty." + ENDPOINT + "}").log(LoggingLevel.DEBUG, "Headers: ${headers}") + .log(LoggingLevel.DEBUG, "Request Body: ${body}").toD("${exchangeProperty." + HOST + "}/${exchangeProperty." + ENDPOINT + + "}" + "?bridgeEndpoint=true" + "&throwExceptionOnFailure=false") + .log(LoggingLevel.DEBUG, "Response body: ${body}"); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java index 6b6a5bf8..2080b774 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java @@ -1,18 +1,28 @@ package org.mifos.processor.bulk.camel.routes; +import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; +import static org.mifos.processor.bulk.camel.config.CamelProperties.OVERRIDE_HEADER; +import static org.mifos.processor.bulk.camel.config.CamelProperties.RESULT_TRANSACTION_LIST; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_LENGTH; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETED_AMOUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FAILED_AMOUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ONGOING_AMOUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TOTAL_AMOUNT; + import com.fasterxml.jackson.databind.MappingIterator; import com.fasterxml.jackson.databind.SequenceWriter; import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.schema.TransactionResult; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.io.*; -import java.util.ArrayList; -import java.util.List; -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @Component public class FileProcessingRoute extends BaseRouteBuilder { @@ -24,13 +34,10 @@ public class FileProcessingRoute extends BaseRouteBuilder { public void configure() { /** - * Parse the [Transaction] array from the csv file - * exchangeInput: [LOCAL_FILE_PATH] the absolute path to the csv file - * exchangeOutput: [TRANSACTION_LIST] containing the list of [Transaction] + * Parse the [Transaction] array from the csv file exchangeInput: [LOCAL_FILE_PATH] the absolute path to the csv + * file exchangeOutput: [TRANSACTION_LIST] containing the list of [Transaction] */ - from("direct:get-transaction-array") - .id("direct:get-transaction-array") - .log("Starting route direct:get-transaction-array") + from("direct:get-transaction-array").id("direct:get-transaction-array").log("Starting route direct:get-transaction-array") .process(exchange -> { Double totalAmount = 0.0; Long failedAmount = 0L; @@ -38,7 +45,8 @@ public void configure() { String filename = exchange.getProperty(LOCAL_FILE_PATH, String.class); CsvSchema schema = CsvSchema.emptySchema().withHeader(); FileReader reader = new FileReader(filename); - MappingIterator readValues = csvMapper.readerWithSchemaFor(Transaction.class).with(schema).readValues(reader); + MappingIterator readValues = csvMapper.readerWithSchemaFor(Transaction.class).with(schema) + .readValues(reader); List transactionList = new ArrayList<>(); while (readValues.hasNext()) { Transaction current = readValues.next(); @@ -49,21 +57,18 @@ public void configure() { exchange.setProperty(TRANSACTION_LIST, transactionList); exchange.setProperty(TRANSACTION_LIST_LENGTH, transactionList.size()); exchange.setProperty(TOTAL_AMOUNT, totalAmount); - exchange.setProperty(ONGOING_AMOUNT, totalAmount); // initially ongoing amount is same as total amount + exchange.setProperty(ONGOING_AMOUNT, totalAmount); // initially ongoing amount is same as total + // amount exchange.setProperty(FAILED_AMOUNT, failedAmount); exchange.setProperty(COMPLETED_AMOUNT, completedAmount); }); /** - * updates the data in local file - * exchangeInput: - * [LOCAL_FILE_PATH] the absolute path to the csv file - * [RESULT_TRANSACTION_LIST] containing the list of [Transaction] - * [OVERRIDE_HEADER] if set to true will override the header or else use the existing once in csv file + * updates the data in local file exchangeInput: [LOCAL_FILE_PATH] the absolute path to the csv file + * [RESULT_TRANSACTION_LIST] containing the list of [Transaction] [OVERRIDE_HEADER] if set to true will override + * the header or else use the existing once in csv file */ - from("direct:update-result-file") - .id("direct:update-result-file") - .log("Starting route direct:update-result-file") + from("direct:update-result-file").id("direct:update-result-file").log("Starting route direct:update-result-file") .process(exchange -> { String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); List transactionList = exchange.getProperty(RESULT_TRANSACTION_LIST, List.class); @@ -72,55 +77,44 @@ public void configure() { Boolean overrideHeader = exchange.getProperty(OVERRIDE_HEADER, Boolean.class); csvWriter(transactionList, TransactionResult.class, csvMapper, overrideHeader, filepath); - }) - .log("Update complete"); + }).log("Update complete"); /** - * updates the data in local file - * exchangeInput: - * [LOCAL_FILE_PATH] the absolute path to the csv file - * [TRANSACTION_LIST] containing the list of [Transaction] - * [OVERRIDE_HEADER] if set to true will override the header or else use the existing once in csv file + * updates the data in local file exchangeInput: [LOCAL_FILE_PATH] the absolute path to the csv file + * [TRANSACTION_LIST] containing the list of [Transaction] [OVERRIDE_HEADER] if set to true will override the + * header or else use the existing once in csv file */ - from("direct:update-file") - .id("direct:update-file") - .log("Starting route direct:update-file") - .to("direct:update-file-v2") + from("direct:update-file").id("direct:update-file").log("Starting route direct:update-file").to("direct:update-file-v2") .log("Update complete"); /** - * this is backward compatible version of update-file route for new CSV schema - * exchangeInput: - * [LOCAL_FILE_PATH] the absolute path to the csv file - * [TRANSACTION_LIST] containing the list of [Transaction] - * [OVERRIDE_HEADER] if set to true will override the header or else use the existing once in csv file + * this is backward compatible version of update-file route for new CSV schema exchangeInput: [LOCAL_FILE_PATH] + * the absolute path to the csv file [TRANSACTION_LIST] containing the list of [Transaction] [OVERRIDE_HEADER] + * if set to true will override the header or else use the existing once in csv file */ - from("direct:update-file-v2") - .id("direct:update-file-v2") - .log("Starting route direct:update-file-v2") - .process(exchange -> { - String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); - List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + from("direct:update-file-v2").id("direct:update-file-v2").log("Starting route direct:update-file-v2").process(exchange -> { + String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); - // getting header - Boolean overrideHeader = exchange.getProperty(OVERRIDE_HEADER, Boolean.class); - CsvSchema csvSchema = csvMapper.schemaFor(Transaction.class); - if (overrideHeader) { - csvSchema = csvSchema.withHeader(); - } else { - csvSchema = csvSchema.withoutHeader(); - } + // getting header + Boolean overrideHeader = exchange.getProperty(OVERRIDE_HEADER, Boolean.class); + CsvSchema csvSchema = csvMapper.schemaFor(Transaction.class); + if (overrideHeader) { + csvSchema = csvSchema.withHeader(); + } else { + csvSchema = csvSchema.withoutHeader(); + } - File file = new File(filepath); - SequenceWriter writer = csvMapper.writerWithSchemaFor(Transaction.class).with(csvSchema).writeValues(file); - for (Transaction transaction: transactionList) { - writer.write(transaction); - } - }); + File file = new File(filepath); + SequenceWriter writer = csvMapper.writerWithSchemaFor(Transaction.class).with(csvSchema).writeValues(file); + for (Transaction transaction : transactionList) { + writer.write(transaction); + } + }); } - private void csvWriter(List data, Class tClass, CsvMapper csvMapper, - boolean overrideHeader, String filepath) throws IOException { + private void csvWriter(List data, Class tClass, CsvMapper csvMapper, boolean overrideHeader, String filepath) + throws IOException { CsvSchema csvSchema = csvMapper.schemaFor(tClass); if (overrideHeader) { csvSchema = csvSchema.withHeader(); @@ -130,7 +124,7 @@ private void csvWriter(List data, Class tClass, CsvMapper csvMapper, File file = new File(filepath); SequenceWriter writer = csvMapper.writerWithSchemaFor(tClass).with(csvSchema).writeValues(file); - for (T object: data) { + for (T object : data) { writer.write(object); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java index 0f0a1542..c4d48b78 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java @@ -1,17 +1,16 @@ package org.mifos.processor.bulk.camel.routes; +import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; + +import java.io.File; +import java.io.FileOutputStream; import org.mifos.processor.bulk.file.FileTransferService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import java.io.File; -import java.io.FileOutputStream; - -import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; -import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; - @Component public class FileRoute extends BaseRouteBuilder { @@ -26,38 +25,30 @@ public class FileRoute extends BaseRouteBuilder { public void configure() throws Exception { /** - * Downloads the file from cloud, stores in local and returns the file path - * Input the file name through exchange variable: [SERVER_FILE_NAME] - * Output the local file path through exchange variable: [LOCAL_FILE_PATH] + * Downloads the file from cloud, stores in local and returns the file path Input the file name through exchange + * variable: [SERVER_FILE_NAME] Output the local file path through exchange variable: [LOCAL_FILE_PATH] */ - from("direct:download-file") - .id("direct:download-file") - .log("Started download-file route") - .process(exchange -> { - String filename = exchange.getProperty(SERVER_FILE_NAME, String.class); - - byte[] csvFile = fileTransferService.downloadFile(filename, bucketName); - File file = new File(filename); - try (FileOutputStream fos = new FileOutputStream(file)) { - fos.write(csvFile); - } - exchange.setProperty(LOCAL_FILE_PATH, file.getAbsolutePath()); - logger.info("File downloaded"); - }); + from("direct:download-file").id("direct:download-file").log("Started download-file route").process(exchange -> { + String filename = exchange.getProperty(SERVER_FILE_NAME, String.class); + + byte[] csvFile = fileTransferService.downloadFile(filename, bucketName); + File file = new File(filename); + try (FileOutputStream fos = new FileOutputStream(file)) { + fos.write(csvFile); + } + exchange.setProperty(LOCAL_FILE_PATH, file.getAbsolutePath()); + logger.info("File downloaded"); + }); /** - * Uploads the file to cloud and returns the file name in cloud - * Input the local file path through exchange variable: [LOCAL_FILE_PATH] - * Output the server file name through exchange variable: [SERVER_FILE_NAME] + * Uploads the file to cloud and returns the file name in cloud Input the local file path through exchange + * variable: [LOCAL_FILE_PATH] Output the server file name through exchange variable: [SERVER_FILE_NAME] */ - from("direct:upload-file") - .id("direct:upload-file") - .log("Uploading file") - .process(exchange -> { - String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); - String serverFileName = fileTransferService.uploadFile(new File(filepath), bucketName); - exchange.setProperty(SERVER_FILE_NAME, serverFileName); - logger.info("Uploaded file: {}", serverFileName); - }); + from("direct:upload-file").id("direct:upload-file").log("Uploading file").process(exchange -> { + String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); + String serverFileName = fileTransferService.uploadFile(new File(filepath), bucketName); + exchange.setProperty(SERVER_FILE_NAME, serverFileName); + logger.info("Uploaded file: {}", serverFileName); + }); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java index 89bf9a1b..f88d8f62 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java @@ -1,20 +1,21 @@ package org.mifos.processor.bulk.camel.routes; -import org.mifos.processor.bulk.schema.Transaction; +import static org.mifos.processor.bulk.camel.config.CamelProperties.OVERRIDE_HEADER; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FORMATTING_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FORMATTING_STANDARD; + +import java.util.List; import org.mifos.processor.bulk.format.Standard; import org.mifos.processor.bulk.format.helper.Mappers; +import org.mifos.processor.bulk.schema.Transaction; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import java.util.List; -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FORMATTING_FAILED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FORMATTING_STANDARD; @Component public class FormattingRoute extends BaseRouteBuilder { - @Autowired private Mappers mappers; @@ -29,47 +30,32 @@ public void configure() throws Exception { formattingStandard = Standard.valueOf(standard); /** - * Base route for kicking off ordering logic. Performs below tasks. - * 1. Downloads the csv form cloud. - * 2. Builds the [Transaction] array using [direct:get-transaction-array] route. - * 3. Format the data based on the configuration provided in application.yaml. @see [Standard.java] - * 4. Update file with the updated data. - * 5. Uploads the updated file in cloud. + * Base route for kicking off ordering logic. Performs below tasks. 1. Downloads the csv form cloud. 2. Builds + * the [Transaction] array using [direct:get-transaction-array] route. 3. Format the data based on the + * configuration provided in application.yaml. @see [Standard.java] 4. Update file with the updated data. 5. + * Uploads the updated file in cloud. */ - from(RouteId.FORMATTING.getValue()) - .id(RouteId.FORMATTING.getValue()) - .log("Starting route " + RouteId.FORMATTING.name()) - .choice() - .when(exchange -> formattingStandard != Standard.DEFAULT) - .to("direct:download-file") - .to("direct:get-transaction-array") + from(RouteId.FORMATTING.getValue()).id(RouteId.FORMATTING.getValue()).log("Starting route " + RouteId.FORMATTING.name()).choice() + .when(exchange -> formattingStandard != Standard.DEFAULT).to("direct:download-file").to("direct:get-transaction-array") .to("direct:format-data") // making sure to override header as well, since data format is now updated - .process(exchange -> exchange.setProperty(OVERRIDE_HEADER, true)) - .to("direct:update-file") - .to("direct:upload-file") - .otherwise() - .log("Skipping formatting since standard is set to DEFAULT") - .end() - .process(exchange -> { + .process(exchange -> exchange.setProperty(OVERRIDE_HEADER, true)).to("direct:update-file").to("direct:upload-file") + .otherwise().log("Skipping formatting since standard is set to DEFAULT").end().process(exchange -> { exchange.setProperty(FORMATTING_FAILED, false); exchange.setProperty(FORMATTING_STANDARD, standard); }); // formatting data based on configuration. Uses [Mappers] for converting data. - from("direct:format-data") - .id("direct:format-data") - .log("Starting route direct:format-data") - .process(exchange -> { - List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); - // replace with switch when multiple standards are added - if (formattingStandard == Standard.GSMA) { - logger.info("Formatting based on {} standard", formattingStandard.name()); - exchange.setProperty(TRANSACTION_LIST, mappers.gsmaMapper.convertList(transactionList)); - } else { - exchange.setProperty(TRANSACTION_LIST, transactionList); - } - }); + from("direct:format-data").id("direct:format-data").log("Starting route direct:format-data").process(exchange -> { + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + // replace with switch when multiple standards are added + if (formattingStandard == Standard.GSMA) { + logger.info("Formatting based on {} standard", formattingStandard.name()); + exchange.setProperty(TRANSACTION_LIST, mappers.gsmaMapper.convertList(transactionList)); + } else { + exchange.setProperty(TRANSACTION_LIST, transactionList); + } + }); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index 1a3eb532..45f95efc 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -1,20 +1,53 @@ package org.mifos.processor.bulk.camel.routes; +import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_ID_HEADER; +import static org.mifos.processor.bulk.camel.config.CamelProperties.EXTERNAL_ENDPOINT; +import static org.mifos.processor.bulk.camel.config.CamelProperties.EXTERNAL_ENDPOINT_FAILED; +import static org.mifos.processor.bulk.camel.config.CamelProperties.IS_PAYMENT_MODE_VALID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; +import static org.mifos.processor.bulk.camel.config.CamelProperties.OVERRIDE_HEADER; +import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYMENT_MODE_TYPE; +import static org.mifos.processor.bulk.camel.config.CamelProperties.RESULT_TRANSACTION_LIST; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_ELEMENT; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_LENGTH; +import static org.mifos.processor.bulk.camel.config.CamelProperties.ZEEBE_VARIABLE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETED_AMOUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DEBULKINGDFSPID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FAILED_AMOUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_SUB_BATCH_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ONGOING_AMOUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYMENT_MODE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.RESULT_FILE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SUB_BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TOTAL_AMOUNT; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Function; import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; -import org.mifos.processor.bulk.config.*; +import org.mifos.processor.bulk.config.ExternalApiPayloadConfig; +import org.mifos.processor.bulk.config.PaymentModeConfiguration; +import org.mifos.processor.bulk.config.PaymentModeMapping; +import org.mifos.processor.bulk.config.PaymentModeType; +import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.schema.TransactionResult; import org.mifos.processor.bulk.utility.Utils; -import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.zeebe.BpmnConfig; import org.mifos.processor.bulk.zeebe.ZeebeProcessStarter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import java.util.*; -import java.util.function.Function; -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @Component public class InitSubBatchRoute extends BaseRouteBuilder { @@ -34,27 +67,19 @@ public class InitSubBatchRoute extends BaseRouteBuilder { @Value("${channel.hostname}") private String ChannelURL; - @Override public void configure() throws Exception { /** - * Base route for kicking off init sub batch logic. Performs below tasks. - * 1. Downloads the csv form cloud. - * 2. Builds the [Transaction] array using [direct:get-transaction-array] route. - * 3. Loops through each transaction and start the respective workflow + * Base route for kicking off init sub batch logic. Performs below tasks. 1. Downloads the csv form cloud. 2. + * Builds the [Transaction] array using [direct:get-transaction-array] route. 3. Loops through each transaction + * and start the respective workflow */ - from(RouteId.INIT_SUB_BATCH.getValue()) - .id(RouteId.INIT_SUB_BATCH.getValue()) - .log("Starting route " + RouteId.INIT_SUB_BATCH.name()) - .to("direct:download-file") - .to("direct:get-transaction-array") - .to("direct:start-workflow-step1"); + from(RouteId.INIT_SUB_BATCH.getValue()).id(RouteId.INIT_SUB_BATCH.getValue()).log("Starting route " + RouteId.INIT_SUB_BATCH.name()) + .to("direct:download-file").to("direct:get-transaction-array").to("direct:start-workflow-step1"); // crates the zeebe variables map and starts the workflow by calling >> direct:start-workflow-step2 - from("direct:start-workflow-step1") - .id("direct:start-flow-step1") - .log("Starting route direct:start-flow-step1") + from("direct:start-workflow-step1").id("direct:start-flow-step1").log("Starting route direct:start-flow-step1") .process(exchange -> { List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); @@ -68,71 +93,47 @@ public void configure() throws Exception { variables.put(ONGOING_AMOUNT, exchange.getProperty(ONGOING_AMOUNT)); variables.put(FAILED_AMOUNT, exchange.getProperty(FAILED_AMOUNT)); variables.put(COMPLETED_AMOUNT, exchange.getProperty(COMPLETED_AMOUNT)); - variables.put(RESULT_FILE, String.format("Result_%s", - exchange.getProperty(SERVER_FILE_NAME))); + variables.put(RESULT_FILE, String.format("Result_%s", exchange.getProperty(SERVER_FILE_NAME))); exchange.setProperty(ZEEBE_VARIABLE, variables); exchange.setProperty(PAYMENT_MODE, transactionList.get(0).getPaymentMode()); + }).to("direct:start-workflow-step2"); - }) - .to("direct:start-workflow-step2"); - - - from("direct:start-workflow-step2") - .id("direct:start-flow-step2") - .log("Starting route direct:start-flow-step2") - .to("direct:validate-payment-mode") - .choice() + from("direct:start-workflow-step2").id("direct:start-flow-step2").log("Starting route direct:start-flow-step2") + .to("direct:validate-payment-mode").choice() // if invalid payment mode - .when(exchangeProperty(IS_PAYMENT_MODE_VALID).isEqualTo(false)) - .to("direct:payment-mode-missing") + .when(exchangeProperty(IS_PAYMENT_MODE_VALID).isEqualTo(false)).to("direct:payment-mode-missing") .setProperty(INIT_SUB_BATCH_FAILED, constant(true)) // else - .otherwise() - .to("direct:start-workflow-step3") - .endChoice(); + .otherwise().to("direct:start-workflow-step3").endChoice(); - from("direct:start-workflow-step3") - .id("direct:start-flow-step3") - .log("Starting route direct:start-flow-step3") - .choice() + from("direct:start-workflow-step3").id("direct:start-flow-step3").log("Starting route direct:start-flow-step3").choice() // if type of payment mode is bulk - .when(exchangeProperty(PAYMENT_MODE_TYPE).isEqualTo(PaymentModeType.BULK)) - .process(exchange -> { + .when(exchangeProperty(PAYMENT_MODE_TYPE).isEqualTo(PaymentModeType.BULK)).process(exchange -> { String paymentMode = exchange.getProperty(PAYMENT_MODE, String.class); PaymentModeMapping mapping = paymentModeConfiguration.getByMode(paymentMode); String tenantName = exchange.getProperty(TENANT_NAME, String.class); - tenantName = mapping.getDebulkingDfspid()==null ? tenantName : mapping.getDebulkingDfspid(); + tenantName = mapping.getDebulkingDfspid() == null ? tenantName : mapping.getDebulkingDfspid(); Map variables = exchange.getProperty(ZEEBE_VARIABLE, Map.class); variables.put(PAYMENT_MODE, paymentMode); - variables.put(DEBULKINGDFSPID, - mapping.getDebulkingDfspid()==null ? tenantName : mapping.getDebulkingDfspid()); + variables.put(DEBULKINGDFSPID, mapping.getDebulkingDfspid() == null ? tenantName : mapping.getDebulkingDfspid()); zeebeProcessStarter.startZeebeWorkflow( - Utils.getBulkConnectorBpmnName(mapping.getEndpoint(), mapping.getId().toLowerCase(), tenantName), - variables); + Utils.getBulkConnectorBpmnName(mapping.getEndpoint(), mapping.getId().toLowerCase(), tenantName), variables); exchange.setProperty(INIT_SUB_BATCH_FAILED, false); }) // if type of payment mode is payment todo // else case or else if case ? - .otherwise() - .loop(simple("${exchangeProperty." + TRANSACTION_LIST_LENGTH + "}")) - .process(exchange -> { + .otherwise().loop(simple("${exchangeProperty." + TRANSACTION_LIST_LENGTH + "}")).process(exchange -> { int index = exchange.getProperty(Exchange.LOOP_INDEX, Integer.class); List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); Transaction transaction = transactionList.get(index); exchange.setProperty(TRANSACTION_LIST_ELEMENT, transaction); - }) - .setHeader("Platform-TenantId", exchangeProperty(TENANT_NAME)) - .to("direct:dynamic-payload-setter") - .to("direct:external-api-call") - .to("direct:external-api-response-handler") - .end() // end loop block + }).setHeader("Platform-TenantId", exchangeProperty(TENANT_NAME)).to("direct:dynamic-payload-setter") + .to("direct:external-api-call").to("direct:external-api-response-handler").end() // end loop block .endChoice(); - from("direct:dynamic-payload-setter") - .id("direct:runtime-payload-test") - .log("Starting route direct:runtime-payload-test") + from("direct:dynamic-payload-setter").id("direct:runtime-payload-test").log("Starting route direct:runtime-payload-test") .process(exchange -> { String mode = exchange.getProperty(PAYMENT_MODE, String.class); Function localPayloadVariable = externalApiPayloadConfig.getApiPayloadSetter(mode); @@ -144,23 +145,16 @@ public void configure() throws Exception { .setBody(simple("${exchangeProperty.body}")); // Loops through each transaction and start the respective workflow - from("direct:external-api-response-handler") - .id("direct:external-api-response-handler") - .log("Starting route direct:external-api-response-handler") - .choice() - .when(header("CamelHttpResponseCode").isEqualTo(200)) + from("direct:external-api-response-handler").id("direct:external-api-response-handler") + .log("Starting route direct:external-api-response-handler").choice().when(header("CamelHttpResponseCode").isEqualTo(200)) .process(exchange -> { logger.info("reached here"); - exchange.setProperty(INIT_SUB_BATCH_FAILED, false);}) - .otherwise() - .process(exchange -> { + exchange.setProperty(INIT_SUB_BATCH_FAILED, false); + }).otherwise().process(exchange -> { exchange.setProperty(INIT_SUB_BATCH_FAILED, true); - }) - .endChoice(); + }).endChoice(); - from("direct:payment-mode-missing") - .id("direct:payment-mode-missing") - .log("Starting route direct:payment-mode-missing") + from("direct:payment-mode-missing").id("direct:payment-mode-missing").log("Starting route direct:payment-mode-missing") .process(exchange -> { String serverFileName = exchange.getProperty(SERVER_FILE_NAME, String.class); String resultFile = String.format("Result_%s", serverFileName); @@ -171,46 +165,32 @@ public void configure() throws Exception { exchange.setProperty(RESULT_FILE, resultFile); }) // setting localfilepath as result file to make sure result file is uploaded - .setProperty(LOCAL_FILE_PATH, exchangeProperty(RESULT_FILE)) - .setProperty(OVERRIDE_HEADER, constant(true)) + .setProperty(LOCAL_FILE_PATH, exchangeProperty(RESULT_FILE)).setProperty(OVERRIDE_HEADER, constant(true)) .process(exchange -> { logger.info("A1 {}", exchange.getProperty(RESULT_FILE)); logger.info("A2 {}", exchange.getProperty(LOCAL_FILE_PATH)); logger.info("A3 {}", exchange.getProperty(OVERRIDE_HEADER)); - }) - .to("direct:update-result-file") - .to("direct:upload-file"); - - - from("direct:external-api-call") - .id("direct:external-api-call") - .log("Starting route direct:external-api-call") - .process(exchange -> { - String paymentMode = exchange.getProperty(PAYMENT_MODE, String.class); - PaymentModeMapping mapping = paymentModeConfiguration.getByMode(paymentMode); - if (mapping == null) { - exchange.setProperty(EXTERNAL_ENDPOINT_FAILED, true); - logger.info("Failed to get the payment mode config, check the configuration for payment mode"); - } else { - exchange.setProperty(EXTERNAL_ENDPOINT_FAILED, false); - exchange.setProperty(EXTERNAL_ENDPOINT, mapping.getEndpoint()); - logger.info("Got the config with routing to endpoint {}", mapping.getEndpoint()); - } - }) - .choice() - .when(exchangeProperty(EXTERNAL_ENDPOINT_FAILED).isEqualTo(false)) + }).to("direct:update-result-file").to("direct:upload-file"); + + from("direct:external-api-call").id("direct:external-api-call").log("Starting route direct:external-api-call").process(exchange -> { + String paymentMode = exchange.getProperty(PAYMENT_MODE, String.class); + PaymentModeMapping mapping = paymentModeConfiguration.getByMode(paymentMode); + if (mapping == null) { + exchange.setProperty(EXTERNAL_ENDPOINT_FAILED, true); + logger.info("Failed to get the payment mode config, check the configuration for payment mode"); + } else { + exchange.setProperty(EXTERNAL_ENDPOINT_FAILED, false); + exchange.setProperty(EXTERNAL_ENDPOINT, mapping.getEndpoint()); + logger.info("Got the config with routing to endpoint {}", mapping.getEndpoint()); + } + }).choice().when(exchangeProperty(EXTERNAL_ENDPOINT_FAILED).isEqualTo(false)) .log(LoggingLevel.DEBUG, "Making API call to endpoint ${exchangeProperty.extEndpoint} and body: ${body}") .setHeader(Exchange.CONTENT_TYPE, constant("application/json")) .setHeader(BATCH_ID_HEADER, simple("${exchangeProperty." + BATCH_ID + "}")) .toD(ChannelURL + "${exchangeProperty.extEndpoint}" + "?bridgeEndpoint=true&throwExceptionOnFailure=false") - .log(LoggingLevel.DEBUG, "Response body: ${body}") - .otherwise() - .endChoice(); - + .log(LoggingLevel.DEBUG, "Response body: ${body}").otherwise().endChoice(); - from("direct:validate-payment-mode") - .id("direct:validate-payment-mode") - .log("Starting route direct:validate-payment-mode") + from("direct:validate-payment-mode").id("direct:validate-payment-mode").log("Starting route direct:validate-payment-mode") .process(exchange -> { String paymentMde = exchange.getProperty(PAYMENT_MODE, String.class); PaymentModeMapping mapping = paymentModeConfiguration.getByMode(paymentMde); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java index 581a17a7..db6d6456 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java @@ -1,52 +1,45 @@ package org.mifos.processor.bulk.camel.routes; -import com.amazonaws.services.dynamodbv2.xspec.L; +import static org.mifos.processor.bulk.camel.config.CamelProperties.FILE_1; +import static org.mifos.processor.bulk.camel.config.CamelProperties.FILE_2; +import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_COMPLETED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_FILE_LIST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_ITERATION; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.RESULT_FILE; + +import java.io.File; +import java.util.List; import org.apache.camel.Exchange; import org.mifos.processor.bulk.utility.Utils; import org.springframework.stereotype.Component; -import java.io.File; -import java.util.List; -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @Component public class MergeBackRoute extends BaseRouteBuilder { - @Override public void configure() throws Exception { /** - * Base route for kicking off merge back logic. Performs below tasks. - * 1. Picks the top two files from the array of files to be merged. - * 2. Merges them into single CSV. - * 3. Uploads the CSV to S3. - * 4. Updated the exchange variables. + * Base route for kicking off merge back logic. Performs below tasks. 1. Picks the top two files from the array + * of files to be merged. 2. Merges them into single CSV. 3. Uploads the CSV to S3. 4. Updated the exchange + * variables. */ - from(RouteId.MERGE_BACK.getValue()) - .id(RouteId.MERGE_BACK.getValue()) - .log("Starting route " + RouteId.MERGE_BACK.name()) - .choice() - .when(exchange -> exchange.getProperty(MERGE_FILE_LIST, List.class).size() == 0) - .log("Nothing to merge") - .setProperty(MERGE_FAILED, constant(false)) - .setProperty(MERGE_COMPLETED, constant(true)) - .when(exchange -> exchange.getProperty(MERGE_FILE_LIST, List.class).size() == 1) - .process(exchange -> { + from(RouteId.MERGE_BACK.getValue()).id(RouteId.MERGE_BACK.getValue()).log("Starting route " + RouteId.MERGE_BACK.name()).choice() + .when(exchange -> exchange.getProperty(MERGE_FILE_LIST, List.class).size() == 0).log("Nothing to merge") + .setProperty(MERGE_FAILED, constant(false)).setProperty(MERGE_COMPLETED, constant(true)) + .when(exchange -> exchange.getProperty(MERGE_FILE_LIST, List.class).size() == 1).process(exchange -> { exchange.setProperty(MERGE_FAILED, false); exchange.setProperty(MERGE_COMPLETED, true); String resultFile = (String) exchange.getProperty(MERGE_FILE_LIST, List.class).get(0); setResultFileProperty(exchange, resultFile); - }) - .otherwise() - .to("direct:start-merge") - .endChoice(); + }).otherwise().to("direct:start-merge").endChoice(); // starts the merge process, merges the file and uploads the file in s3 - from("direct:start-merge") - .id("direct:start-merge") - .log("Starting route direct:start-merge") - .to("direct:download-file-to-be-merged") + from("direct:start-merge").id("direct:start-merge").log("Starting route direct:start-merge").to("direct:download-file-to-be-merged") .process(exchange -> { String file1 = exchange.getProperty(FILE_1, String.class); String file2 = exchange.getProperty(FILE_2, String.class); @@ -58,16 +51,13 @@ public void configure() throws Exception { } if (exchange.getProperty(MERGE_ITERATION, Integer.class) == 1) { // generate new name for merged file in case of first iteration - String newFileName = System.currentTimeMillis() + "_" + - exchange.getProperty(BATCH_ID, String.class) + ".csv"; + String newFileName = System.currentTimeMillis() + "_" + exchange.getProperty(BATCH_ID, String.class) + ".csv"; new File(mergedFile).renameTo(new File(newFileName)); exchange.setProperty(LOCAL_FILE_PATH, newFileName); } else { exchange.setProperty(LOCAL_FILE_PATH, mergedFile); } - }) - .to("direct:upload-file") - .process(exchange -> { + }).to("direct:upload-file").process(exchange -> { String mergedFileServerName = exchange.getProperty(SERVER_FILE_NAME, String.class); List mergeList = exchange.getProperty(MERGE_FILE_LIST, List.class); String first = mergeList.remove(0); @@ -76,7 +66,6 @@ public void configure() throws Exception { log.info("Merged files {} and {}", first, second); mergeList.add(0, mergedFileServerName); - if (mergeList.size() == 1) { exchange.setProperty(MERGE_FAILED, false); exchange.setProperty(MERGE_COMPLETED, true); @@ -93,21 +82,15 @@ public void configure() throws Exception { }); // downloads the two files(using FIFO access pattern) from s3 which is to be merged. - from("direct:download-file-to-be-merged") - .id("direct:download-file-to-be-merged") - .log("Starting route direct:download-file-to-be-merged") - .log("Downloading files to be merged") - .process(exchange -> { + from("direct:download-file-to-be-merged").id("direct:download-file-to-be-merged") + .log("Starting route direct:download-file-to-be-merged").log("Downloading files to be merged").process(exchange -> { List mergeList = exchange.getProperty(MERGE_FILE_LIST, List.class); exchange.setProperty(SERVER_FILE_NAME, mergeList.get(0)); - }) - .to("direct:download-file") // downloading first file - .setProperty(FILE_1, exchangeProperty(LOCAL_FILE_PATH)) - .process(exchange -> { + }).to("direct:download-file") // downloading first file + .setProperty(FILE_1, exchangeProperty(LOCAL_FILE_PATH)).process(exchange -> { List mergeList = exchange.getProperty(MERGE_FILE_LIST, List.class); exchange.setProperty(SERVER_FILE_NAME, mergeList.get(1)); - }) - .to("direct:download-file") // downloading second file + }).to("direct:download-file") // downloading second file .setProperty(FILE_2, exchangeProperty(LOCAL_FILE_PATH)); } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/OperationsAuthRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/OperationsAuthRoute.java index 6f31b074..7da3f864 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/OperationsAuthRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/OperationsAuthRoute.java @@ -1,13 +1,14 @@ package org.mifos.processor.bulk.camel.routes; +import static org.mifos.processor.bulk.camel.config.CamelProperties.OPS_APP_ACCESS_TOKEN; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; + +import java.util.HashMap; +import java.util.Map; import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; import org.apache.camel.model.dataformat.JsonLibrary; import org.springframework.stereotype.Component; -import java.util.HashMap; -import java.util.Map; -import static org.mifos.processor.bulk.camel.config.CamelProperties.OPS_APP_ACCESS_TOKEN; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; @Component public class OperationsAuthRoute extends BaseRouteBuilder { @@ -15,57 +16,42 @@ public class OperationsAuthRoute extends BaseRouteBuilder { @Override public void configure() throws Exception { - from("rest:get:test/auth") - .to("direct:get-access-token"); + from("rest:get:test/auth").to("direct:get-access-token"); /** * Error handling route */ - from("direct:access-token-error") - .id("access-token-error") - .process(exchange -> { - logger.error("Error while fetching Access Token from server: " + exchange.getIn().getBody()); - }); + from("direct:access-token-error").id("access-token-error").process(exchange -> { + logger.error("Error while fetching Access Token from server: " + exchange.getIn().getBody()); + }); /** * Save Access Token to AccessTokenStore */ - from("direct:access-token-save") - .id("access-token-save") - .unmarshal().json(JsonLibrary.Jackson, HashMap.class) - .process(exchange -> { - // TODO: Figure out access token storage if required - Map jsonObject = exchange.getIn().getBody(HashMap.class); - exchange.setProperty(OPS_APP_ACCESS_TOKEN, jsonObject.get("access_token")); - logger.debug("Saved Access Token: " + exchange.getProperty(OPS_APP_ACCESS_TOKEN, String.class)); - exchange.getIn().setBody(jsonObject.toString()); - }); + from("direct:access-token-save").id("access-token-save").unmarshal().json(JsonLibrary.Jackson, HashMap.class).process(exchange -> { + // TODO: Figure out access token storage if required + Map jsonObject = exchange.getIn().getBody(HashMap.class); + exchange.setProperty(OPS_APP_ACCESS_TOKEN, jsonObject.get("access_token")); + logger.debug("Saved Access Token: " + exchange.getProperty(OPS_APP_ACCESS_TOKEN, String.class)); + exchange.getIn().setBody(jsonObject.toString()); + }); /** * Fetch Access Token from SLCB */ getBaseExternalApiRequestRouteDefinition("access-token-fetch", HttpRequestMethod.POST) .setHeader(Exchange.REST_HTTP_QUERY, - simpleF("username=%s&password=%s&grant_type=%s", - operationsAppConfig.username, operationsAppConfig.password, "password")) + simpleF("username=%s&password=%s&grant_type=%s", operationsAppConfig.username, operationsAppConfig.password, + "password")) .setHeader("Authorization", constant("Basic Y2xpZW50Og==")) .setHeader("Platform-TenantId", simple("${exchangeProperty." + TENANT_ID + "}")) - .toD(operationsAppConfig.authUrl + "?bridgeEndpoint=true") - .log(LoggingLevel.INFO, "Auth response: \n\n ${body}"); - + .toD(operationsAppConfig.authUrl + "?bridgeEndpoint=true").log(LoggingLevel.INFO, "Auth response: \n\n ${body}"); /** * Access Token check validity and return value */ - from("direct:get-access-token") - .id("get-access-token") - .to("direct:access-token-fetch") - .choice() - .when(header("CamelHttpResponseCode").isEqualTo("200")) - .log("Access Token Fetch Successful") - .to("direct:access-token-save") - .otherwise() - .log("Access Token Fetch Unsuccessful") - .to("direct:access-token-error"); + from("direct:get-access-token").id("get-access-token").to("direct:access-token-fetch").choice() + .when(header("CamelHttpResponseCode").isEqualTo("200")).log("Access Token Fetch Successful").to("direct:access-token-save") + .otherwise().log("Access Token Fetch Unsuccessful").to("direct:access-token-error"); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java index b504b4cb..71c2c565 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java @@ -1,14 +1,15 @@ package org.mifos.processor.bulk.camel.routes; -import org.mifos.processor.bulk.schema.Transaction; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERED_BY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERING_FAILED; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERED_BY; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERING_FAILED; +import org.mifos.processor.bulk.schema.Transaction; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; @Component public class OrderingRoute extends BaseRouteBuilder { @@ -16,78 +17,68 @@ public class OrderingRoute extends BaseRouteBuilder { @Value("${config.ordering.field}") private String orderingField; - @Override public void configure() { /** - * Base route for kicking off ordering logic. Performs below tasks. - * 1. Downloads the csv form cloud. - * 2. Builds the [Transaction] array using CsvMapper. - * 3. Re-order the array generated in step1 based on [orderingField]. - * 4. Update file with the updated data. - * 5. Uploads the updated file in cloud. + * Base route for kicking off ordering logic. Performs below tasks. 1. Downloads the csv form cloud. 2. Builds + * the [Transaction] array using CsvMapper. 3. Re-order the array generated in step1 based on [orderingField]. + * 4. Update file with the updated data. 5. Uploads the updated file in cloud. */ - from(RouteId.ORDERING.getValue()) - .id(RouteId.ORDERING.getValue()) - .log("Starting route " + RouteId.ORDERING.name()) - .to("direct:download-file") - .to("direct:get-transaction-array") - .to("direct:order-data") - .to("direct:update-file") - .to("direct:upload-file") - .process(exchange -> { + from(RouteId.ORDERING.getValue()).id(RouteId.ORDERING.getValue()).log("Starting route " + RouteId.ORDERING.name()) + .to("direct:download-file").to("direct:get-transaction-array").to("direct:order-data").to("direct:update-file") + .to("direct:upload-file").process(exchange -> { exchange.setProperty(ORDERING_FAILED, false); exchange.setProperty(ORDERED_BY, orderingField); }); // re-order the array of [Transaction] based on [orderingField] - from("direct:order-data") - .id("direct:order-data") - .log("Starting route direct:order-data") - .process(exchange -> { - List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); - HashMap> stringListHashMap = new HashMap<>(); - transactionList.forEach((transaction) -> { - String key; - switch (orderingField) { - case "id": - key = "" + transaction.getId(); - break; - case "request_id": - key = transaction.getRequestId(); - break; - case "account_number": - key = transaction.getAccountNumber(); - break; - case "payee_identifier": - key = transaction.getPayeeIdentifier(); - break; - case "amount": - key = transaction.getAmount(); - break; - case "currency": - key = transaction.getCurrency(); - break; - case "note": - key = transaction.getNote(); - break; - default: - key = transaction.getPaymentMode(); - break; - } + from("direct:order-data").id("direct:order-data").log("Starting route direct:order-data").process(exchange -> { + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + HashMap> stringListHashMap = new HashMap<>(); + transactionList.forEach((transaction) -> { + String key; + switch (orderingField) { + case "id": + key = "" + transaction.getId(); + break; + case "request_id": + key = transaction.getRequestId(); + break; + case "account_number": + key = transaction.getAccountNumber(); + break; + case "payee_identifier": + key = transaction.getPayeeIdentifier(); + break; + case "amount": + key = transaction.getAmount(); + break; + case "currency": + key = transaction.getCurrency(); + break; + case "note": + key = transaction.getNote(); + break; + default: + key = transaction.getPaymentMode(); + break; + } + + if (stringListHashMap.containsKey(key)) { + stringListHashMap.get(key).add(transaction); + } else { + stringListHashMap.put(key, new ArrayList() { - if (stringListHashMap.containsKey(key)) { - stringListHashMap.get(key).add(transaction); - } else { - stringListHashMap.put(key, new ArrayList() {{ - add(transaction); - }}); + { + add(transaction); } }); - transactionList.clear(); - stringListHashMap.forEach((s, transactions) -> transactionList.addAll(transactions)); - exchange.setProperty(TRANSACTION_LIST, transactionList); - }); + } + }); + transactionList.clear(); + stringListHashMap.forEach((s, transactions) -> transactionList.addAll(transactions)); + exchange.setProperty(TRANSACTION_LIST, transactionList); + }); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 85685d60..df97d7aa 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -1,10 +1,51 @@ package org.mifos.processor.bulk.camel.routes; +import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_REQUEST_TYPE; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.APPROVAL_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BULK_NOTIF_FAILURE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BULK_NOTIF_SUCCESS; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_URL; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_THRESHOLD; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_THRESHOLD_CHECK_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_VALIDITY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FORMATTING_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_CALLBACK_RETRY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_STATUS_RETRY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.NOTE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERING_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASES; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASE_COUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.THRESHOLD_DELAY; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; +import javax.activation.DataHandler; +import javax.mail.internet.MimeBodyPart; import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; - import org.apache.tika.Tika; - import org.json.JSONObject; import org.mifos.processor.bulk.file.FileTransferService; import org.mifos.processor.bulk.utility.PhaseUtils; @@ -18,18 +59,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import javax.activation.DataHandler; -import javax.mail.internet.MimeBodyPart; -import java.io.*; -import java.nio.charset.StandardCharsets; -import java.util.*; -import java.util.stream.Collectors; - -import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_REQUEST_TYPE; -import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; - - @Component public class ProcessorStartRoute extends BaseRouteBuilder { @@ -82,99 +111,70 @@ public void configure() { private void setup() { - from("rest:POST:/batchtransactions") - .id("rest:POST:/batchtransactions") - .log("Starting route rest:POST:/batchtransactions") - .to("direct:validate-file") - .choice() - .when(header("CamelHttpResponseCode").isNotEqualTo("200")) - .log(LoggingLevel.ERROR, "File upload failed") - .otherwise() - .unmarshal().mimeMultipart("multipart/*") - .process(exchange -> { + from("rest:POST:/batchtransactions").id("rest:POST:/batchtransactions").log("Starting route rest:POST:/batchtransactions") + .to("direct:validate-file").choice().when(header("CamelHttpResponseCode").isNotEqualTo("200")) + .log(LoggingLevel.ERROR, "File upload failed").otherwise().unmarshal().mimeMultipart("multipart/*").process(exchange -> { String batchId = UUID.randomUUID().toString(); - exchange.setProperty(BATCH_ID,batchId); - - }) - .unmarshal().mimeMultipart("multipart/*") - .to("direct:validateFileSyncResponse") - .choice() - .when(header("CamelHttpResponseCode").isNotEqualTo("200")) - .log(LoggingLevel.ERROR, "File upload failed") - .otherwise() - .unmarshal().mimeMultipart("multipart/*") - .wireTap("direct:executeBatch") - .to("direct:pollingOutput") - .endChoice() - .endChoice(); - - - - from("rest:POST:/bulk/transfer/{requestId}/{fileName}") - .unmarshal().mimeMultipart("multipart/*") - .to("direct:validate-tenant") + exchange.setProperty(BATCH_ID, batchId); + + }).unmarshal().mimeMultipart("multipart/*").to("direct:validateFileSyncResponse").choice() + .when(header("CamelHttpResponseCode").isNotEqualTo("200")).log(LoggingLevel.ERROR, "File upload failed").otherwise() + .unmarshal().mimeMultipart("multipart/*").wireTap("direct:executeBatch").to("direct:pollingOutput").endChoice().endChoice(); + + from("rest:POST:/bulk/transfer/{requestId}/{fileName}").unmarshal().mimeMultipart("multipart/*").to("direct:validate-tenant") .process(exchange -> { - String fileName = System.currentTimeMillis() + "_" + exchange.getIn().getHeader("fileName", String.class); + String fileName = System.currentTimeMillis() + "_" + exchange.getIn().getHeader("fileName", String.class); String requestId = exchange.getIn().getHeader("requestId", String.class); String purpose = exchange.getIn().getHeader("purpose", String.class); String batchId = UUID.randomUUID().toString(); - exchange.setProperty(BATCH_ID,batchId); + exchange.setProperty(BATCH_ID, batchId); exchange.setProperty(FILE_NAME, fileName); exchange.setProperty(REQUEST_ID, requestId); exchange.setProperty(PURPOSE, purpose); - }) - .wireTap("direct:start-batch-process-csv") - .to("direct:pollingOutput"); - - from("direct:validate-tenant") - .id("direct:validate-tenant") - .log("Validating tenant") - .process(exchange -> { - String tenantName = exchange.getIn().getHeader("Platform-TenantId", String.class); - // validation is disabled for now - /*if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { - throw new Exception("Invalid tenant value."); - }*/ - exchange.setProperty(TENANT_NAME, tenantName); - }) - .setHeader("Content-Type", constant("application/json;charset=UTF-8")) - .log("Completed route direct:validate-tenant"); - - from("direct:start-batch-process-csv") - .id("direct:start-batch-process-csv") - .log("Starting route direct:start-batch-process-csv") + }).wireTap("direct:start-batch-process-csv").to("direct:pollingOutput"); + + from("direct:validate-tenant").id("direct:validate-tenant").log("Validating tenant").process(exchange -> { + String tenantName = exchange.getIn().getHeader("Platform-TenantId", String.class); + // validation is disabled for now + /* + * if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { throw new + * Exception("Invalid tenant value."); } + */ + exchange.setProperty(TENANT_NAME, tenantName); + }).setHeader("Content-Type", constant("application/json;charset=UTF-8")).log("Completed route direct:validate-tenant"); + + from("direct:start-batch-process-csv").id("direct:start-batch-process-csv").log("Starting route direct:start-batch-process-csv") .process(exchange -> { String fileName = exchange.getProperty(FILE_NAME, String.class); String requestId = exchange.getProperty(REQUEST_ID, String.class); String purpose = exchange.getProperty(PURPOSE, String.class); - String batchId = exchange.getProperty(BATCH_ID,String.class); + String batchId = exchange.getProperty(BATCH_ID, String.class); String note = null; - if (purpose == null || purpose.isEmpty()) { purpose = "test payment"; } logger.info("\n\n Filename: {}", fileName); - logger.info("\n\n BatchId: {} ",batchId); + logger.info("\n\n BatchId: {} ", batchId); InputStream csvData = exchange.getIn().getBody(InputStream.class); - File file = setupFile(csvData,fileName); - logger.debug("File length {}",file.length()); + File file = setupFile(csvData, fileName); + logger.debug("File length {}", file.length()); boolean verifyData = verifyData(file); logger.debug("Data verification result {}", verifyData); - if(!verifyData){ + if (!verifyData) { note = "Invalid data in file data processing stopped"; - + } String nm = fileTransferService.uploadFile(file, bucketName); logger.info("File uploaded {}", nm); - //extracting and setting callback Url + // extracting and setting callback Url String callbackUrl = exchange.getIn().getHeader("X-Callback-URL", String.class); - exchange.setProperty(CALLBACK_URL,callbackUrl); + exchange.setProperty(CALLBACK_URL, callbackUrl); List phases = phaseUtils.getValues(); logger.debug(phases.toString()); @@ -185,17 +185,16 @@ private void setup() { variables.put(REQUEST_ID, requestId); variables.put(PURPOSE, purpose); variables.put(TENANT_ID, exchange.getProperty(TENANT_NAME)); - variables.put(CALLBACK_URL,callbackUrl); - variables.put(PHASES,phases); - variables.put(PHASE_COUNT,phases.size()); - variables.put(NOTE,note); + variables.put(CALLBACK_URL, callbackUrl); + variables.put(PHASES, phases); + variables.put(PHASE_COUNT, phases.size()); + variables.put(NOTE, note); setConfigProperties(variables); JSONObject response = new JSONObject(); try { - String tenantSpecificWorkflowId = workflowId.replace("{dfspid}", - exchange.getProperty(TENANT_NAME).toString()); + String tenantSpecificWorkflowId = workflowId.replace("{dfspid}", exchange.getProperty(TENANT_NAME).toString()); String txnId = zeebeProcessStarter.startZeebeWorkflow(tenantSpecificWorkflowId, "", variables); if (txnId == null || txnId.isEmpty()) { response.put("errorCode", 500); @@ -214,26 +213,18 @@ private void setup() { exchange.getIn().setBody(response.toString()); - }) - .log("Completed route direct:start-batch-process-csv"); - + }).log("Completed route direct:start-batch-process-csv"); - from("direct:start-batch-process-raw") - .id("direct:start-batch-process-raw") - .log("Starting route direct:start-batch-process-raw") + from("direct:start-batch-process-raw").id("direct:start-batch-process-raw").log("Starting route direct:start-batch-process-raw") .process(exchange -> { JSONObject response = new JSONObject(); response.put("batch_id", UUID.randomUUID().toString()); response.put("request_id", UUID.randomUUID().toString()); response.put("status", "queued"); exchange.getIn().setBody(response.toString()); - }) - .log("Completed route direct:start-batch-process-raw"); + }).log("Completed route direct:start-batch-process-raw"); - from("direct:executeBatch") - .id("direct:executeBatch") - .log("Starting route direct:executeBatch") - .to("direct:validate-tenant") + from("direct:executeBatch").id("direct:executeBatch").log("Starting route direct:executeBatch").to("direct:validate-tenant") .process(exchange -> { String filename = exchange.getIn().getHeader("filename", String.class); String requestId = exchange.getIn().getHeader("X-CorrelationID", String.class); @@ -243,81 +234,62 @@ private void setup() { exchange.setProperty(REQUEST_ID, requestId); exchange.setProperty(PURPOSE, purpose); exchange.setProperty(BATCH_REQUEST_TYPE, type); - }) - .choice() - .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("raw")) + }).choice().when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("raw")) .to("direct:start-batch-process-raw") - .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("csv")) - .unmarshal().mimeMultipart("multipart/*") - .to("direct:start-batch-process-csv") - .otherwise() + .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("csv")).unmarshal() + .mimeMultipart("multipart/*").to("direct:start-batch-process-csv").otherwise() .setBody(exchange -> getUnsupportedTypeJson(exchange.getProperty(BATCH_REQUEST_TYPE, String.class)).toString()) .log("Completed execution of route rest:POST:/batchtransactions"); - from("direct:pollingOutput") - .id("direct:pollingOutput") - .log("Started pollingOutput route") - .process(exchange -> { - JSONObject json = new JSONObject(); - json.put("PollingPath", "/batch/Summary/"+exchange.getProperty(BATCH_ID)); - json.put("SuggestedCallbackSeconds",pollApiTimer); - exchange.getIn().setBody(json.toString()); + from("direct:pollingOutput").id("direct:pollingOutput").log("Started pollingOutput route").process(exchange -> { + JSONObject json = new JSONObject(); + json.put("PollingPath", "/batch/Summary/" + exchange.getProperty(BATCH_ID)); + json.put("SuggestedCallbackSeconds", pollApiTimer); + exchange.getIn().setBody(json.toString()); - }); + }); - from("direct:validateFileSyncResponse") - .id("direct:validateFileSyncResponse") - .log("Starting route direct:validateFileSyncResponse") + from("direct:validateFileSyncResponse").id("direct:validateFileSyncResponse").log("Starting route direct:validateFileSyncResponse") .process(exchange -> { - String fileName = System.currentTimeMillis() + "_" + exchange.getIn().getHeader("fileName", String.class); + String fileName = System.currentTimeMillis() + "_" + exchange.getIn().getHeader("fileName", String.class); InputStream csvData = exchange.getIn().getBody(InputStream.class); - File file = setupFile(csvData,fileName); + File file = setupFile(csvData, fileName); int fileSize = (int) file.length(); - if (fileSize > csvSize) { + if (fileSize > csvSize) { setErrorResponse(exchange, 400, "File too big", - "The file uploaded is too big. " + - "Please upload a file and try again."); - } - else if(!verifyCsv(file)){ + "The file uploaded is too big. " + "Please upload a file and try again."); + } else if (!verifyCsv(file)) { setErrorResponse(exchange, 400, "Invalid file structure", - "The file uploaded contains wrong structure." + - " Please upload correct file columns and try again."); - } - else { + "The file uploaded contains wrong structure." + " Please upload correct file columns and try again."); + } else { logger.debug("Filename: {}", fileName); setResponse(exchange, 200); } - }) - .log("Completed route direct:validateFileSyncResponse"); - - from("direct:validate-file") - .id("direct:validate-file") - .log("Starting route direct:validate-file") - .process(exchange -> { - System.setProperty("mail.mime.multipart.ignoreexistingboundaryparameter", "true"); - System.setProperty("mail.mime.multipart.allowempty", "true"); - InputStream inputStreams = exchange.getIn().getBody(InputStream.class); - MimeBodyPart mimeMessage = new MimeBodyPart(inputStreams); - DataHandler dh = mimeMessage.getDataHandler(); - logger.debug("File name: {} ",dh.getName()); - Tika tika = new Tika(); - String fileType = tika.detect(dh.getName()); - logger.debug("File type: {} ", fileType); - if (dh.getName() == null || dh.getName().isEmpty() ) { - setErrorResponse(exchange, 400, "File not uploaded", - "There was no fie uploaded with the request. " + - "Please upload a file and try again."); - } - else if (!fileType.equalsIgnoreCase("text/csv")) { - setErrorResponse(exchange, 400, "Broken file", - "The file uploaded is broken as it has a different extension. " + - "Please upload a csv file and try again."); - } - else{ - setResponse(exchange, 200);} + }).log("Completed route direct:validateFileSyncResponse"); + + from("direct:validate-file").id("direct:validate-file").log("Starting route direct:validate-file").process(exchange -> { + System.setProperty("mail.mime.multipart.ignoreexistingboundaryparameter", "true"); + System.setProperty("mail.mime.multipart.allowempty", "true"); + InputStream inputStreams = exchange.getIn().getBody(InputStream.class); + MimeBodyPart mimeMessage = new MimeBodyPart(inputStreams); + DataHandler dh = mimeMessage.getDataHandler(); + logger.debug("File name: {} ", dh.getName()); + Tika tika = new Tika(); + String fileType = tika.detect(dh.getName()); + logger.debug("File type: {} ", fileType); + if (dh.getName() == null || dh.getName().isEmpty()) { + setErrorResponse(exchange, 400, "File not uploaded", + "There was no fie uploaded with the request. " + "Please upload a file and try again."); + } else if (!fileType.equalsIgnoreCase("text/csv")) { + setErrorResponse(exchange, 400, "Broken file", + "The file uploaded is broken as it has a different extension. " + "Please upload a csv file and try again."); + } else { + setResponse(exchange, 200); + } - });} + }); + } private boolean verifyData(File file) throws IOException { InputStream ips = new FileInputStream(file); @@ -331,39 +303,38 @@ private boolean verifyData(File file) throws IOException { logger.debug("Row invalid {} {}", row.length, columnNames.size()); return false; } - if(!verifyRow(row)) + if (!verifyRow(row)) { return false; + } } return true; } + private boolean verifyRow(String[] row) { - for (int i = 1; i < row.length; i++) { - row[i] = row[i].trim(); - if (row[i].equalsIgnoreCase("MSISDN")) { - int j = row[i].indexOf("MSISDN"); - if (!(j == row.length) ){ - if(!row[j+ 1].matches("^[0-9]*$")) { - logger.debug("MSISDN invalid"); - return false; - } - } - } - else if (row[i].contains("amount")){ - int j = row[i].indexOf("amount"); - if(!row[j].matches("^[0-9]*$")){ - logger.debug("Amount invalid"); + for (int i = 1; i < row.length; i++) { + row[i] = row[i].trim(); + if (row[i].equalsIgnoreCase("MSISDN")) { + int j = row[i].indexOf("MSISDN"); + if (!(j == row.length)) { + if (!row[j + 1].matches("^[0-9]*$")) { + logger.debug("MSISDN invalid"); return false; } - } + } else if (row[i].contains("amount")) { + int j = row[i].indexOf("amount"); + if (!row[j].matches("^[0-9]*$")) { + logger.debug("Amount invalid"); + return false; + } + } - return true; } + return true; + } private File setupFile(InputStream csvStream, String fileName) throws IOException { - String csvData = new BufferedReader( - new InputStreamReader(csvStream, StandardCharsets.UTF_8)) - .lines() + String csvData = new BufferedReader(new InputStreamReader(csvStream, StandardCharsets.UTF_8)).lines() .collect(Collectors.joining("\n")); File file = new File(fileName); file.setWritable(true); @@ -374,7 +345,6 @@ private File setupFile(InputStream csvStream, String fileName) throws IOExceptio return file; } - private boolean verifyCsv(File csvData) throws IOException { BufferedReader br = new BufferedReader(new FileReader(csvData)); String header = br.readLine(); @@ -384,8 +354,8 @@ private boolean verifyCsv(File csvData) throws IOException { logger.info("Columns in the csv file are {}", Arrays.toString(columns)); } int i = 0; - while(i < columns.length) { - if(columnNames.contains(columns[i])) { + while (i < columns.length) { + if (columnNames.contains(columns[i])) { logger.debug("Column name {} is at index {} ", columns[i], columnNames.indexOf(columns[i])); i++; @@ -396,23 +366,22 @@ private boolean verifyCsv(File csvData) throws IOException { return true; } - private void setErrorResponse(Exchange exchange,int responseCode, String errorInfo, String errorDescription) { + private void setErrorResponse(Exchange exchange, int responseCode, String errorInfo, String errorDescription) { // TODO Auto-generated method stub JSONObject json = new JSONObject(); json.put("Error Information: ", errorInfo); - json.put("Error Description : ",errorDescription); + json.put("Error Description : ", errorDescription); exchange.getIn().setHeader(Exchange.HTTP_RESPONSE_CODE, responseCode); exchange.getIn().setBody(json.toString()); exchange.setProperty("body", json); logger.error("Error response is {}", json); } - private void setResponse(Exchange exchange,int responseCode){ + + private void setResponse(Exchange exchange, int responseCode) { exchange.getIn().setHeader(Exchange.HTTP_RESPONSE_CODE, responseCode); } - - private Map setConfigProperties(Map variables) { variables.put(PARTY_LOOKUP_ENABLED, workerConfig.isPartyLookUpWorkerEnabled); variables.put(APPROVAL_ENABLED, workerConfig.isApprovalWorkerEnabled); @@ -424,9 +393,9 @@ private Map setConfigProperties(Map variables) { variables.put(MAX_STATUS_RETRY, maxThresholdCheckRetry); variables.put(COMPLETION_THRESHOLD, completionThreshold); variables.put(THRESHOLD_DELAY, Utils.getZeebeTimerValue(thresholdCheckDelay)); - variables.put(BULK_NOTIF_SUCCESS,false); - variables.put(BULK_NOTIF_FAILURE,false); - variables.put(MAX_CALLBACK_RETRY,maxCallbackRetry); + variables.put(BULK_NOTIF_SUCCESS, false); + variables.put(BULK_NOTIF_FAILURE, false); + variables.put(MAX_CALLBACK_RETRY, maxCallbackRetry); return variables; } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java b/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java index 74819193..9af8fae8 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java @@ -2,15 +2,9 @@ public enum RouteId { - PARTY_LOOKUP("direct:partyLookup"), - APPROVAL("direct:approval"), - ORDERING("direct:ordering"), - SPLITTING("direct:splitting"), - FORMATTING("direct:formatting"), - BATCH_STATUS("direct:batchStatus"), - SEND_CALLBACK("direct:sendCallback"), - MERGE_BACK("direct:mergeSubBatch"), - INIT_SUB_BATCH("direct:init-sub-batches"); + PARTY_LOOKUP("direct:partyLookup"), APPROVAL("direct:approval"), ORDERING("direct:ordering"), SPLITTING("direct:splitting"), FORMATTING( + "direct:formatting"), BATCH_STATUS("direct:batchStatus"), SEND_CALLBACK( + "direct:sendCallback"), MERGE_BACK("direct:mergeSubBatch"), INIT_SUB_BATCH("direct:init-sub-batches"); private final String value; diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java index 4a594f55..de537c0b 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java @@ -1,22 +1,18 @@ package org.mifos.processor.bulk.camel.routes; +import static org.mifos.processor.bulk.camel.config.CamelProperties.IS_BATCH_READY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.IS_SAMPLE_READY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SAMPLED_TX_IDS; import com.google.gson.Gson; -import io.camunda.zeebe.client.ZeebeClient; -import org.apache.camel.builder.RouteBuilder; -import org.json.JSONArray; -import org.json.JSONObject; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; - -import static org.mifos.processor.bulk.camel.config.CamelProperties.IS_BATCH_READY; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; +import org.json.JSONObject; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; @Component public class Routes extends BaseRouteBuilder { @@ -32,18 +28,15 @@ public void configure() throws Exception { private void routeCheckTransactions() { String id = "check-transactions"; - from("direct:" + id) - .id(id) - .log("Fetching transaction details") - //set request params - .toD(operationsAppConfig.batchTransactionEndpoint) - .process(exchange -> { + from("direct:" + id).id(id).log("Fetching transaction details") + // set request params + .toD(operationsAppConfig.batchTransactionEndpoint).process(exchange -> { // get response body JSONObject transfers = new JSONObject(exchange.getIn().getBody(String.class)); int totalTransactions = transfers.length(); int successfulTransactions = 0; - for (Iterator it = transfers.keys(); it.hasNext(); ) { + for (Iterator it = transfers.keys(); it.hasNext();) { String transactionId = it.next(); String transactionStatus = transfers.getString(transactionId); if (transactionStatus.equals("COMPLETED")) { @@ -53,60 +46,51 @@ private void routeCheckTransactions() { HashMap newVariables = new HashMap<>(); // check successful transactions >= x% - if (((double)successfulTransactions / totalTransactions) >= minimumSuccessfulTxRatio) { + if (((double) successfulTransactions / totalTransactions) >= minimumSuccessfulTxRatio) { newVariables.put(IS_SAMPLE_READY, true); } else { newVariables.put(IS_SAMPLE_READY, false); } - zeebeClient.newSetVariablesCommand(Long.parseLong(exchange.getProperty(BATCH_ID).toString())) - .variables(newVariables) - .send() - .join(); + zeebeClient.newSetVariablesCommand(Long.parseLong(exchange.getProperty(BATCH_ID).toString())).variables(newVariables) + .send().join(); }); } private void routeSampleTransactions() { String id = "sample-transactions"; - from("direct:" + id) - .id(id) - .log("Fetching transaction details") - .process(exchange -> { - exchange.getIn().setHeader("batchId", exchange.getProperty(BATCH_ID)); - }) - .toD(operationsAppConfig.batchTransactionEndpoint) - .process(exchange -> { - // get response body + from("direct:" + id).id(id).log("Fetching transaction details").process(exchange -> { + exchange.getIn().setHeader("batchId", exchange.getProperty(BATCH_ID)); + }).toD(operationsAppConfig.batchTransactionEndpoint).process(exchange -> { + // get response body - // check if batch is ready for sampling - if (exchange.getProperty(IS_BATCH_READY, String.class).equals("false")) { - return; - } - // sample transactions - JSONObject transfers = new JSONObject(exchange.getIn().getBody(String.class)); - final ArrayList successfulTransactionIds = new ArrayList<>(); - final ArrayList sampledTransactionIds = new ArrayList<>(); - for (Iterator it = transfers.keys(); it.hasNext(); ) { - String transactionId = it.next(); - String transactionStatus = transfers.getString(transactionId); - if (transactionStatus.equals("COMPLETED")) { - successfulTransactionIds.add(transactionId); - } - } - Collections.shuffle(successfulTransactionIds); - int sampleSize = (int) (successfulTransactionIds.size() * 0.9); - for (int i = 0; i < sampleSize; i++) { - sampledTransactionIds.add(successfulTransactionIds.get(i)); - } - HashMap newVariables = new HashMap<>(); - newVariables.put(SAMPLED_TX_IDS, new Gson().toJson(sampledTransactionIds)); + // check if batch is ready for sampling + if (exchange.getProperty(IS_BATCH_READY, String.class).equals("false")) { + return; + } + // sample transactions + JSONObject transfers = new JSONObject(exchange.getIn().getBody(String.class)); + final ArrayList successfulTransactionIds = new ArrayList<>(); + final ArrayList sampledTransactionIds = new ArrayList<>(); + for (Iterator it = transfers.keys(); it.hasNext();) { + String transactionId = it.next(); + String transactionStatus = transfers.getString(transactionId); + if (transactionStatus.equals("COMPLETED")) { + successfulTransactionIds.add(transactionId); + } + } + Collections.shuffle(successfulTransactionIds); + int sampleSize = (int) (successfulTransactionIds.size() * 0.9); + for (int i = 0; i < sampleSize; i++) { + sampledTransactionIds.add(successfulTransactionIds.get(i)); + } + HashMap newVariables = new HashMap<>(); + newVariables.put(SAMPLED_TX_IDS, new Gson().toJson(sampledTransactionIds)); - // store the sampled transaction ids in zeebe variable - zeebeClient.newSetVariablesCommand(Long.parseLong(exchange.getProperty(BATCH_ID).toString())) - .variables(newVariables) - .send() - .join(); + // store the sampled transaction ids in zeebe variable + zeebeClient.newSetVariablesCommand(Long.parseLong(exchange.getProperty(BATCH_ID).toString())).variables(newVariables).send() + .join(); - }); + }); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java index c87e81b2..6376f4c3 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java @@ -1,78 +1,69 @@ package org.mifos.processor.bulk.camel.routes; +import static org.mifos.processor.bulk.camel.config.CamelProperties.CALLBACK_RESPONSE_CODE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_RETRY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_SUCCESS; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_URL; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_RATE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_CODE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_DESCRIPTION; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_CALLBACK_RETRY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASES; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASE_COUNT; + +import java.util.List; import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import java.util.List; - -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; - @Component public class SendCallbackRoute extends BaseRouteBuilder { - @Value("${callback.url}") private String callbackUrl; @Override public void configure() throws Exception { - from("rest:get:test/send/callback") - .to(RouteId.SEND_CALLBACK.getValue()); + from("rest:get:test/send/callback").to(RouteId.SEND_CALLBACK.getValue()); /** - * Base route for kicking off callback. Performs below tasks. - * Sends Callback to the set url - * Checks of response code is anything not 2xx then retries + * Base route for kicking off callback. Performs below tasks. Sends Callback to the set url Checks of response + * code is anything not 2xx then retries */ - from(RouteId.SEND_CALLBACK.getValue()) - .id(RouteId.SEND_CALLBACK.getValue()) - .log("Starting route " + RouteId.SEND_CALLBACK.name()) - .log("Sending callback for Batch Processing") - .setBody(exchange -> { - String body = String.format - ("The Batch Aggregation API was complete with : %s" - , exchange.getProperty(COMPLETION_RATE).toString()); + from(RouteId.SEND_CALLBACK.getValue()).id(RouteId.SEND_CALLBACK.getValue()).log("Starting route " + RouteId.SEND_CALLBACK.name()) + .log("Sending callback for Batch Processing").setBody(exchange -> { + String body = String.format("The Batch Aggregation API was complete with : %s", + exchange.getProperty(COMPLETION_RATE).toString()); callbackUrl = exchange.getProperty(CALLBACK_URL).toString(); return body; - }) - .toD("${header.callbackUrl}?bridgeEndpoint=true&throwExceptionOnFailure=false") - .choice() - .when(header("CamelHttpResponseCode").startsWith("2")) - .log(LoggingLevel.INFO, "Callback sending was successful") + }).toD("${header.callbackUrl}?bridgeEndpoint=true&throwExceptionOnFailure=false").choice() + .when(header("CamelHttpResponseCode").startsWith("2")).log(LoggingLevel.INFO, "Callback sending was successful") .process(exchange -> { List phases = (List) exchange.getProperty(PHASES); - exchange.setProperty(CALLBACK_RESPONSE_CODE, exchange.getIn() - .getHeader(Exchange.HTTP_RESPONSE_CODE)); + exchange.setProperty(CALLBACK_RESPONSE_CODE, exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE)); exchange.setProperty(CALLBACK_RETRY, 1); exchange.setProperty(CALLBACK_SUCCESS, true); - eliminatePhases(exchange,phases, (Integer)exchange.getProperty(PHASE_COUNT), - (Integer)exchange.getProperty(COMPLETION_RATE)); + eliminatePhases(exchange, phases, (Integer) exchange.getProperty(PHASE_COUNT), + (Integer) exchange.getProperty(COMPLETION_RATE)); - }) - .otherwise() - .log(LoggingLevel.ERROR, "Callback request was unsuccessful") - .process(exchange -> { + }).otherwise().log(LoggingLevel.ERROR, "Callback request was unsuccessful").process(exchange -> { if (exchange.getProperty(CALLBACK_RETRY).equals(exchange.getProperty(MAX_CALLBACK_RETRY))) { List phases = (List) exchange.getProperty(PHASES); logger.info("Retry Exhausted, setting Callback as Failed"); - eliminatePhases(exchange,phases, (Integer)exchange.getProperty(PHASE_COUNT), - (Integer)exchange.getProperty(COMPLETION_RATE)); - exchange.setProperty(CALLBACK_RESPONSE_CODE, exchange.getIn() - .getHeader(Exchange.HTTP_RESPONSE_CODE)); + eliminatePhases(exchange, phases, (Integer) exchange.getProperty(PHASE_COUNT), + (Integer) exchange.getProperty(COMPLETION_RATE)); + exchange.setProperty(CALLBACK_RESPONSE_CODE, exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE)); } else { int retry = (int) exchange.getProperty(CALLBACK_RETRY); retry++; logger.info("Retry Left {}, Setting Callback as Failed and Retrying...", ((int) exchange.getProperty(MAX_CALLBACK_RETRY) - retry)); exchange.setProperty(CALLBACK_RETRY, retry); - exchange.setProperty(CALLBACK_RESPONSE_CODE, exchange.getIn() - .getHeader(Exchange.HTTP_RESPONSE_CODE)); + exchange.setProperty(CALLBACK_RESPONSE_CODE, exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE)); } exchange.setProperty(CALLBACK_SUCCESS, false); @@ -80,20 +71,18 @@ public void configure() throws Exception { exchange.setProperty(ERROR_CODE, exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE)); }); - //for temporary callback simulation - from("rest:post:/simulate") - .log("Reached Simulation"); + // for temporary callback simulation + from("rest:post:/simulate").log("Reached Simulation"); } - public void eliminatePhases(Exchange exchange, List phases, int phaseCount, int completionRate){ + public void eliminatePhases(Exchange exchange, List phases, int phaseCount, int completionRate) { int i = 0; - while(phases.size() > 0 && phases.size() > i){ - if(phases.get(i) <= completionRate) - phases.remove(i); + while (phases.size() > 0 && phases.size() > i) { + if (phases.get(i) <= completionRate) phases.remove(i); i++; } - exchange.setProperty(PHASES,phases); - exchange.setProperty(PHASE_COUNT,phaseCount); + exchange.setProperty(PHASES, phases); + exchange.setProperty(PHASE_COUNT, phaseCount); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java index b0373adb..ff868ad5 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java @@ -1,14 +1,20 @@ package org.mifos.processor.bulk.camel.routes; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; +import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_SUB_BATCH_FILE_NAME_ARRAY; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_COUNT; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_CREATED; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_FILE_ARRAY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_FAILED; + import java.io.BufferedReader; import java.io.FileReader; import java.io.FileWriter; import java.util.ArrayList; import java.util.List; -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_FAILED; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; @Component public class SplittingRoute extends BaseRouteBuilder { @@ -20,77 +26,60 @@ public class SplittingRoute extends BaseRouteBuilder { public void configure() throws Exception { /** - * Base route for starting the splitting process. Refer below routes for more info - * 1. direct:create-sub-batch-file - * 2. direct:upload-sub-batch-file + * Base route for starting the splitting process. Refer below routes for more info 1. + * direct:create-sub-batch-file 2. direct:upload-sub-batch-file */ - from(RouteId.SPLITTING.getValue()) - .id(RouteId.SPLITTING.getValue()) - .log("Starting route " + RouteId.SPLITTING.name()) - .to("direct:download-file") - .to("direct:create-sub-batch-file") - .choice() - .when(exchange -> exchange.getProperty(SUB_BATCH_CREATED, Boolean.class)) - .to("direct:upload-sub-batch-file") - .otherwise() - .log("No sub batch created, so skipping upload") - .end() - .process(exchange -> exchange.setProperty(SPLITTING_FAILED, false)); + from(RouteId.SPLITTING.getValue()).id(RouteId.SPLITTING.getValue()).log("Starting route " + RouteId.SPLITTING.name()) + .to("direct:download-file").to("direct:create-sub-batch-file").choice() + .when(exchange -> exchange.getProperty(SUB_BATCH_CREATED, Boolean.class)).to("direct:upload-sub-batch-file").otherwise() + .log("No sub batch created, so skipping upload").end().process(exchange -> exchange.setProperty(SPLITTING_FAILED, false)); - //Creates the sub-batch CSVs - from("direct:create-sub-batch-file") - .id("direct:create-sub-batch-file") - .log("Creating sub-batch file") - .process(exchange -> { - String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); - BufferedReader reader = new BufferedReader(new FileReader(filepath)); - String header = reader.readLine() + System.lineSeparator(); - List lines = new ArrayList<>(); - String line = null; - while ((line = reader.readLine()) != null) { - lines.add(line); - } - reader.close(); + // Creates the sub-batch CSVs + from("direct:create-sub-batch-file").id("direct:create-sub-batch-file").log("Creating sub-batch file").process(exchange -> { + String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); + BufferedReader reader = new BufferedReader(new FileReader(filepath)); + String header = reader.readLine() + System.lineSeparator(); + List lines = new ArrayList<>(); + String line = null; + while ((line = reader.readLine()) != null) { + lines.add(line); + } + reader.close(); - if (lines.size() <= subBatchSize) { - exchange.setProperty(SUB_BATCH_CREATED, false); - exchange.setProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, new ArrayList()); - logger.info("Skipping creating sub batch, as batch size is less than configured sub-batch size"); - return; - } + if (lines.size() <= subBatchSize) { + exchange.setProperty(SUB_BATCH_CREATED, false); + exchange.setProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, new ArrayList()); + logger.info("Skipping creating sub batch, as batch size is less than configured sub-batch size"); + return; + } - List subBatchFile = new ArrayList<>(); - int subBatchCount = 1; - for (int i = 0; i < lines.size(); i+=subBatchSize) { - String filename = System.currentTimeMillis() + "_" + "sub-batch-"+subBatchCount+".csv"; - FileWriter writer = new FileWriter(filename); - writer.write(header); - for (int j = i; j < Math.min(i + subBatchSize, lines.size()); j++) { - writer.write(lines.get(j) + System.lineSeparator()); - } - writer.close(); - logger.info("Created sub-batch with file name {}", filename); - subBatchFile.add(filename); - subBatchCount++; - } - exchange.setProperty(SUB_BATCH_FILE_ARRAY, subBatchFile); - exchange.setProperty(SUB_BATCH_COUNT, subBatchFile.size()); - exchange.setProperty(SUB_BATCH_CREATED, true); - exchange.setProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, new ArrayList()); - }); + List subBatchFile = new ArrayList<>(); + int subBatchCount = 1; + for (int i = 0; i < lines.size(); i += subBatchSize) { + String filename = System.currentTimeMillis() + "_" + "sub-batch-" + subBatchCount + ".csv"; + FileWriter writer = new FileWriter(filename); + writer.write(header); + for (int j = i; j < Math.min(i + subBatchSize, lines.size()); j++) { + writer.write(lines.get(j) + System.lineSeparator()); + } + writer.close(); + logger.info("Created sub-batch with file name {}", filename); + subBatchFile.add(filename); + subBatchCount++; + } + exchange.setProperty(SUB_BATCH_FILE_ARRAY, subBatchFile); + exchange.setProperty(SUB_BATCH_COUNT, subBatchFile.size()); + exchange.setProperty(SUB_BATCH_CREATED, true); + exchange.setProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, new ArrayList()); + }); // Iterate through each CSVs of sub-batches and uploads in cloud - from("direct:upload-sub-batch-file") - .id("direct:upload-sub-batch-file") - .log("Starting upload of sub-batch file") - .loopDoWhile(exchange -> exchange.getProperty(SUB_BATCH_FILE_ARRAY, List.class).size() > 0) - .process(exchange -> { + from("direct:upload-sub-batch-file").id("direct:upload-sub-batch-file").log("Starting upload of sub-batch file") + .loopDoWhile(exchange -> exchange.getProperty(SUB_BATCH_FILE_ARRAY, List.class).size() > 0).process(exchange -> { List subBatchFile = exchange.getProperty(SUB_BATCH_FILE_ARRAY, List.class); exchange.setProperty(LOCAL_FILE_PATH, subBatchFile.remove(0)); exchange.setProperty(SUB_BATCH_FILE_ARRAY, subBatchFile); - }) - .to("direct:upload-file") - .process(exchange -> { + }).to("direct:upload-file").process(exchange -> { String serverFilename = exchange.getProperty(SERVER_FILE_NAME, String.class); List serverSubBatchFile = exchange.getProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, List.class); serverSubBatchFile.add(serverFilename); diff --git a/src/main/java/org/mifos/processor/bulk/config/ExternalApiPayloadConfig.java b/src/main/java/org/mifos/processor/bulk/config/ExternalApiPayloadConfig.java index a586e208..613e98d6 100644 --- a/src/main/java/org/mifos/processor/bulk/config/ExternalApiPayloadConfig.java +++ b/src/main/java/org/mifos/processor/bulk/config/ExternalApiPayloadConfig.java @@ -1,18 +1,16 @@ package org.mifos.processor.bulk.config; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; +import javax.annotation.PostConstruct; import lombok.Getter; import org.apache.camel.Exchange; -import org.apache.camel.Processor; import org.mifos.processor.bulk.camel.processor.GsmaApiPayload; import org.mifos.processor.bulk.camel.processor.MojaloopApiPayload; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; -import javax.annotation.PostConstruct; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - @Configuration @Getter public class ExternalApiPayloadConfig { @@ -30,7 +28,7 @@ public class ExternalApiPayloadConfig { @PostConstruct private void registerApiProcessor() { - for (PaymentModeMapping paymentMode: paymentModeConfiguration.getMappings()) { + for (PaymentModeMapping paymentMode : paymentModeConfiguration.getMappings()) { if (paymentMode.getId().equalsIgnoreCase("gsma")) { payloadMap.put(paymentMode.getId(), gsmaApiPayload); } else if (paymentMode.getId().equalsIgnoreCase("mojaloop")) { diff --git a/src/main/java/org/mifos/processor/bulk/config/PaymentModeConfiguration.java b/src/main/java/org/mifos/processor/bulk/config/PaymentModeConfiguration.java index 97b0960a..4597f031 100644 --- a/src/main/java/org/mifos/processor/bulk/config/PaymentModeConfiguration.java +++ b/src/main/java/org/mifos/processor/bulk/config/PaymentModeConfiguration.java @@ -1,13 +1,11 @@ package org.mifos.processor.bulk.config; +import java.util.ArrayList; +import java.util.List; import lombok.Getter; import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.List; @Configuration @ConfigurationProperties(prefix = "payment-mode") @@ -18,10 +16,7 @@ public class PaymentModeConfiguration { private List mappings = new ArrayList<>(); public PaymentModeMapping getByMode(String paymentMode) { - return getMappings().stream() - .filter(p -> p.getId().equalsIgnoreCase(paymentMode)) - .findFirst() - .orElse(null); + return getMappings().stream().filter(p -> p.getId().equalsIgnoreCase(paymentMode)).findFirst().orElse(null); } } diff --git a/src/main/java/org/mifos/processor/bulk/config/PaymentModeMapping.java b/src/main/java/org/mifos/processor/bulk/config/PaymentModeMapping.java index 5895b52c..0d9d4663 100644 --- a/src/main/java/org/mifos/processor/bulk/config/PaymentModeMapping.java +++ b/src/main/java/org/mifos/processor/bulk/config/PaymentModeMapping.java @@ -7,6 +7,8 @@ @Setter public class PaymentModeMapping { - private String id, endpoint, debulkingDfspid; + private String id; + private String endpoint; + private String debulkingDfspid; private PaymentModeType type; } diff --git a/src/main/java/org/mifos/processor/bulk/config/PaymentModeType.java b/src/main/java/org/mifos/processor/bulk/config/PaymentModeType.java index 6fdb6b72..bda7a171 100644 --- a/src/main/java/org/mifos/processor/bulk/config/PaymentModeType.java +++ b/src/main/java/org/mifos/processor/bulk/config/PaymentModeType.java @@ -5,8 +5,7 @@ @Getter public enum PaymentModeType { - PAYMENT("PAYMENT"), - BULK("BULK"); + PAYMENT("PAYMENT"), BULK("BULK"); private String modeType; diff --git a/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java b/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java index 7b68358c..85b17abc 100644 --- a/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java +++ b/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java @@ -5,6 +5,9 @@ import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; import com.amazonaws.util.IOUtils; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -12,13 +15,10 @@ import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; - @Service @Qualifier("awsStorage") public class AwsFileTransferImpl implements FileTransferService { + private Logger logger = LoggerFactory.getLogger(this.getClass()); @Autowired diff --git a/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java b/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java index ffa93d8a..98067c81 100644 --- a/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java +++ b/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java @@ -2,6 +2,10 @@ import com.azure.storage.blob.BlobClientBuilder; import com.azure.storage.blob.models.BlobProperties; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -10,18 +14,11 @@ import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; - @Service @Qualifier("azureStorage") -@ConditionalOnProperty( - value="cloud.azure.enabled", - havingValue = "true") +@ConditionalOnProperty(value = "cloud.azure.enabled", havingValue = "true") public class AzureFileTransferImpl implements FileTransferService { + private Logger logger = LoggerFactory.getLogger(this.getClass()); @Autowired @@ -56,7 +53,7 @@ public String uploadFile(File file, String bucketName) { @Override public byte[] downloadFile(String fileName, String bucketName) { try { - File temp = new File("/temp/"+fileName); + File temp = new File("/temp/" + fileName); BlobProperties properties = client.containerName(bucketName).blobName(fileName).buildClient().downloadToFile(temp.getPath()); byte[] content = Files.readAllBytes(Paths.get(temp.getPath())); temp.delete(); diff --git a/src/main/java/org/mifos/processor/bulk/file/FileTransferService.java b/src/main/java/org/mifos/processor/bulk/file/FileTransferService.java index a9789087..b4c3a6df 100644 --- a/src/main/java/org/mifos/processor/bulk/file/FileTransferService.java +++ b/src/main/java/org/mifos/processor/bulk/file/FileTransferService.java @@ -1,10 +1,9 @@ package org.mifos.processor.bulk.file; +import java.io.File; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; -import java.io.File; - @Service public interface FileTransferService { diff --git a/src/main/java/org/mifos/processor/bulk/file/config/AwsStorageConfig.java b/src/main/java/org/mifos/processor/bulk/file/config/AwsStorageConfig.java index 8b8ef1f4..58f04c6a 100644 --- a/src/main/java/org/mifos/processor/bulk/file/config/AwsStorageConfig.java +++ b/src/main/java/org/mifos/processor/bulk/file/config/AwsStorageConfig.java @@ -23,14 +23,10 @@ public class AwsStorageConfig { private String region; @Bean - @ConditionalOnProperty( - value="cloud.aws.enabled", - havingValue = "true") + @ConditionalOnProperty(value = "cloud.aws.enabled", havingValue = "true") public AmazonS3 s3Client() { AWSCredentials credentials = new BasicAWSCredentials(accessKey, accessSecret); - return AmazonS3ClientBuilder.standard() - .withCredentials(new AWSStaticCredentialsProvider(credentials)) - .withRegion(region).build(); + return AmazonS3ClientBuilder.standard().withCredentials(new AWSStaticCredentialsProvider(credentials)).withRegion(region).build(); } } diff --git a/src/main/java/org/mifos/processor/bulk/file/config/AzureStorageConfig.java b/src/main/java/org/mifos/processor/bulk/file/config/AzureStorageConfig.java index 1dc91e49..f7ffa998 100644 --- a/src/main/java/org/mifos/processor/bulk/file/config/AzureStorageConfig.java +++ b/src/main/java/org/mifos/processor/bulk/file/config/AzureStorageConfig.java @@ -1,6 +1,6 @@ package org.mifos.processor.bulk.file.config; -import com.azure.storage.blob.*; +import com.azure.storage.blob.BlobClientBuilder; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; @@ -13,9 +13,7 @@ public class AzureStorageConfig { String connectionString; @Bean - @ConditionalOnProperty( - value="cloud.azure.enabled", - havingValue = "true") + @ConditionalOnProperty(value = "cloud.azure.enabled", havingValue = "true") public BlobClientBuilder getClient() { BlobClientBuilder client = new BlobClientBuilder(); client.connectionString(connectionString); diff --git a/src/main/java/org/mifos/processor/bulk/format/Standard.java b/src/main/java/org/mifos/processor/bulk/format/Standard.java index 11d6e8de..394670fa 100644 --- a/src/main/java/org/mifos/processor/bulk/format/Standard.java +++ b/src/main/java/org/mifos/processor/bulk/format/Standard.java @@ -2,8 +2,7 @@ public enum Standard { - GSMA(StandardValue.GSMA), - DEFAULT(StandardValue.DEFAULT); + GSMA(StandardValue.GSMA), DEFAULT(StandardValue.DEFAULT); private final String value; @@ -16,4 +15,3 @@ public String getValue() { } } - diff --git a/src/main/java/org/mifos/processor/bulk/format/StandardValue.java b/src/main/java/org/mifos/processor/bulk/format/StandardValue.java index a10e4ff6..9d86d9c8 100644 --- a/src/main/java/org/mifos/processor/bulk/format/StandardValue.java +++ b/src/main/java/org/mifos/processor/bulk/format/StandardValue.java @@ -1,7 +1,8 @@ package org.mifos.processor.bulk.format; public class StandardValue { - public final static String GSMA = "GSMA"; - public final static String DEFAULT = "DEFAULT"; + public static final String GSMA = "GSMA"; + + public static final String DEFAULT = "DEFAULT"; } diff --git a/src/main/java/org/mifos/processor/bulk/format/helper/BaseMapper.java b/src/main/java/org/mifos/processor/bulk/format/helper/BaseMapper.java index 0481fed5..ef311429 100644 --- a/src/main/java/org/mifos/processor/bulk/format/helper/BaseMapper.java +++ b/src/main/java/org/mifos/processor/bulk/format/helper/BaseMapper.java @@ -1,9 +1,8 @@ package org.mifos.processor.bulk.format.helper; -import org.mifos.processor.bulk.schema.CsvSchema; - import java.util.ArrayList; import java.util.List; +import org.mifos.processor.bulk.schema.CsvSchema; public abstract class BaseMapper implements Mapper { diff --git a/src/main/java/org/mifos/processor/bulk/format/helper/GSMAMapper.java b/src/main/java/org/mifos/processor/bulk/format/helper/GSMAMapper.java index ca924cf5..3cb4f427 100644 --- a/src/main/java/org/mifos/processor/bulk/format/helper/GSMAMapper.java +++ b/src/main/java/org/mifos/processor/bulk/format/helper/GSMAMapper.java @@ -6,6 +6,7 @@ @Component public class GSMAMapper extends BaseMapper { + @Override public GSMATransaction convert(Transaction object) { GSMATransaction gsmaTransaction = new GSMATransaction(); diff --git a/src/main/java/org/mifos/processor/bulk/format/helper/Mapper.java b/src/main/java/org/mifos/processor/bulk/format/helper/Mapper.java index 665785fc..ba259dc0 100644 --- a/src/main/java/org/mifos/processor/bulk/format/helper/Mapper.java +++ b/src/main/java/org/mifos/processor/bulk/format/helper/Mapper.java @@ -1,22 +1,24 @@ package org.mifos.processor.bulk.format.helper; -import org.mifos.processor.bulk.schema.CsvSchema; -import org.springframework.stereotype.Service; - import java.util.List; +import org.mifos.processor.bulk.schema.CsvSchema; public interface Mapper { /** * Use for converting the object of type [FROM] to an object of type [TO] - * @param object of type [FROM] + * + * @param object + * of type [FROM] * @return object of type [To] */ public TO convert(FROM object); /** * Use for converting the list of objects of type [FROM] to an object of type [TO] - * @param objects lost of object of type [FROM] + * + * @param objects + * lost of object of type [FROM] * @return objects of type [To] */ public List convertList(List objects); diff --git a/src/main/java/org/mifos/processor/bulk/kafka/Consumers.java b/src/main/java/org/mifos/processor/bulk/kafka/Consumers.java index ce3a78db..09e64067 100644 --- a/src/main/java/org/mifos/processor/bulk/kafka/Consumers.java +++ b/src/main/java/org/mifos/processor/bulk/kafka/Consumers.java @@ -1,7 +1,23 @@ package org.mifos.processor.bulk.kafka; +import static org.mifos.connector.common.mojaloop.type.InitiatorType.CONSUMER; +import static org.mifos.connector.common.mojaloop.type.Scenario.TRANSFER; +import static org.mifos.connector.common.mojaloop.type.TransactionRole.PAYER; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.GSMA_CHANNEL_REQUEST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INITIATOR_FSPID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.IS_RTP_REQUEST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_ID_TYPE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_FSPID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TRANSACTION_TYPE; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.HashMap; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; import org.mifos.connector.common.channel.dto.TransactionChannelRequestDTO; import org.mifos.connector.common.gsma.dto.GSMATransaction; import org.mifos.connector.common.gsma.dto.GsmaParty; @@ -17,27 +33,10 @@ import org.springframework.kafka.annotation.KafkaListener; import org.springframework.stereotype.Service; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.mifos.connector.common.mojaloop.type.InitiatorType.CONSUMER; -import static org.mifos.connector.common.mojaloop.type.Scenario.TRANSFER; -import static org.mifos.connector.common.mojaloop.type.TransactionRole.PAYER; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.GSMA_CHANNEL_REQUEST; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INITIATOR_FSPID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.IS_RTP_REQUEST; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_ID_TYPE; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_FSPID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TRANSACTION_TYPE; - @Service +@Slf4j public class Consumers { - @Value("${bpmn.flows.international-remittance-payer}") private String internationalRemittancePayer; @@ -49,8 +48,7 @@ public class Consumers { @KafkaListener(topics = "${kafka.topic.gsma.name}", groupId = "group_id") public void listenTopicGsma(String message) throws JsonProcessingException { - System.out.println("Received Message in topic GSMA and group group_id: " + message); - + log.debug("Received Message in topic GSMA and group group_id: {}", message); TransactionOlder transaction = objectMapper.readValue((String) message, TransactionOlder.class); String tenantId = "ibank-usa"; @@ -65,9 +63,9 @@ public void listenTopicGsma(String message) throws JsonProcessingException { GsmaParty debitParty = new GsmaParty(); debitParty.setKey("msisdn"); debitParty.setValue(transaction.getAccountNumber()); - gsmaChannelRequest.setCreditParty(new GsmaParty[]{creditParty}); - gsmaChannelRequest.setDebitParty(new GsmaParty[]{debitParty}); -// gsmaChannelRequest.setInternationalTransferInformation().setReceivingAmount(gsmaChannelRequest.getAmount()); + gsmaChannelRequest.setCreditParty(new GsmaParty[] { creditParty }); + gsmaChannelRequest.setDebitParty(new GsmaParty[] { debitParty }); + // gsmaChannelRequest.setInternationalTransferInformation().setReceivingAmount(gsmaChannelRequest.getAmount()); TransactionChannelRequestDTO channelRequest = new TransactionChannelRequestDTO(); // Fineract Object Party payee = new Party(new PartyIdInfo(IdentifierType.MSISDN, transaction.getAccountNumber())); @@ -96,7 +94,8 @@ public void listenTopicGsma(String message) throws JsonProcessingException { String tenantSpecificBpmn = internationalRemittancePayer.replace("{dfspid}", tenantId); channelRequest.setTransactionType(transactionType); - PartyIdInfo requestedParty = (boolean)extraVariables.get(IS_RTP_REQUEST) ? channelRequest.getPayer().getPartyIdInfo() : channelRequest.getPayee().getPartyIdInfo(); + PartyIdInfo requestedParty = (boolean) extraVariables.get(IS_RTP_REQUEST) ? channelRequest.getPayer().getPartyIdInfo() + : channelRequest.getPayee().getPartyIdInfo(); extraVariables.put(PARTY_ID_TYPE, requestedParty.getPartyIdType()); extraVariables.put(PARTY_ID, requestedParty.getPartyIdentifier()); @@ -104,15 +103,14 @@ public void listenTopicGsma(String message) throws JsonProcessingException { extraVariables.put(PARTY_LOOKUP_FSPID, gsmaChannelRequest.getReceivingLei()); extraVariables.put(INITIATOR_FSPID, gsmaChannelRequest.getRequestingLei()); - String transactionId = zeebeProcessStarter.startZeebeWorkflow(tenantSpecificBpmn, - objectMapper.writeValueAsString(channelRequest), + String transactionId = zeebeProcessStarter.startZeebeWorkflow(tenantSpecificBpmn, objectMapper.writeValueAsString(channelRequest), extraVariables); - System.out.println("GSMA Transaction Started with: " + transactionId); + log.debug("GSMA Transaction Started with:{} ", transactionId); } @KafkaListener(topics = "${kafka.topic.slcb.name}", groupId = "group_id") public void listenTopicSlcb(String message) { - System.out.println("Received Message in topic SLCB and group group_id: " + message); + log.debug("Received Message in topic SLCB and group group_id:{} ", message); } } diff --git a/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaConsumerConfig.java b/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaConsumerConfig.java index 5b689a39..e9694b89 100644 --- a/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaConsumerConfig.java +++ b/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaConsumerConfig.java @@ -1,5 +1,7 @@ package org.mifos.processor.bulk.kafka.config; +import java.util.HashMap; +import java.util.Map; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.serialization.StringDeserializer; import org.springframework.beans.factory.annotation.Value; @@ -10,9 +12,6 @@ import org.springframework.kafka.core.ConsumerFactory; import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import java.util.HashMap; -import java.util.Map; - @EnableKafka @Configuration public class KafkaConsumerConfig { diff --git a/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaProducerConfig.java b/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaProducerConfig.java index 4ef5d8d9..11b9aa2a 100644 --- a/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaProducerConfig.java +++ b/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaProducerConfig.java @@ -1,5 +1,7 @@ package org.mifos.processor.bulk.kafka.config; +import java.util.HashMap; +import java.util.Map; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.StringSerializer; import org.springframework.beans.factory.annotation.Value; @@ -9,9 +11,6 @@ import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.core.ProducerFactory; -import java.util.HashMap; -import java.util.Map; - @Configuration public class KafkaProducerConfig { diff --git a/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaTopicConfig.java b/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaTopicConfig.java index 625ddee2..7d2f2ff7 100644 --- a/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaTopicConfig.java +++ b/src/main/java/org/mifos/processor/bulk/kafka/config/KafkaTopicConfig.java @@ -1,5 +1,7 @@ package org.mifos.processor.bulk.kafka.config; +import java.util.HashMap; +import java.util.Map; import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.kafka.clients.admin.NewTopic; import org.springframework.beans.factory.annotation.Value; @@ -7,11 +9,9 @@ import org.springframework.context.annotation.Configuration; import org.springframework.kafka.core.KafkaAdmin; -import java.util.HashMap; -import java.util.Map; - @Configuration public class KafkaTopicConfig { + @Value(value = "${kafka.bootstrapAddress}") private String bootstrapAddress; diff --git a/src/main/java/org/mifos/processor/bulk/schema/AccountLookupResponseDTO.java b/src/main/java/org/mifos/processor/bulk/schema/AccountLookupResponseDTO.java index 5427d135..edadd569 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/AccountLookupResponseDTO.java +++ b/src/main/java/org/mifos/processor/bulk/schema/AccountLookupResponseDTO.java @@ -1,18 +1,18 @@ package org.mifos.processor.bulk.schema; +import java.io.Serializable; +import java.util.List; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; -import java.io.Serializable; -import java.util.List; - @Getter @Setter @NoArgsConstructor @AllArgsConstructor public class AccountLookupResponseDTO implements Serializable { + private String requestId; private String payeeIdentity; private List paymentModalityList; diff --git a/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java b/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java index 33479aa0..26a25536 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java +++ b/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java @@ -1,13 +1,12 @@ package org.mifos.processor.bulk.schema; import java.math.BigDecimal; -import java.util.Date; public class BatchDTO { - private String batch_id; + private String batchId; - private String request_id; + private String requestId; private Long total; @@ -29,7 +28,7 @@ public class BatchDTO { private String notes; - private String created_at; + private String createdAt; private String status; @@ -37,12 +36,13 @@ public class BatchDTO { private String purpose; - public BatchDTO() { - } + public BatchDTO() {} - public BatchDTO(String batchId, String requestId, Long totalTransactions, Long ongoing, Long failed, Long completed, BigDecimal total_amount, BigDecimal completed_amount, BigDecimal ongoing_amount, BigDecimal failed_amount, String result_file, String note) { - this.batch_id = batchId; - this.request_id = requestId; + public BatchDTO(String batchId, String requestId, Long totalTransactions, Long ongoing, Long failed, Long completed, + BigDecimal total_amount, BigDecimal completed_amount, BigDecimal ongoing_amount, BigDecimal failed_amount, String result_file, + String note) { + this.batchId = batchId; + this.requestId = requestId; this.total = totalTransactions; this.ongoing = ongoing; this.failed = failed; @@ -55,9 +55,11 @@ public BatchDTO(String batchId, String requestId, Long totalTransactions, Long o this.notes = note; } - public BatchDTO(String batch_id, String request_id, Long total, Long ongoing, Long failed, Long successful, BigDecimal totalAmount, BigDecimal successfulAmount, BigDecimal pendingAmount, BigDecimal failedAmount, String file, String notes, String created_at, String status, String modes, String purpose) { - this.batch_id = batch_id; - this.request_id = request_id; + public BatchDTO(String batch_id, String request_id, Long total, Long ongoing, Long failed, Long successful, BigDecimal totalAmount, + BigDecimal successfulAmount, BigDecimal pendingAmount, BigDecimal failedAmount, String file, String notes, String created_at, + String status, String modes, String purpose) { + this.batchId = batch_id; + this.requestId = request_id; this.total = total; this.ongoing = ongoing; this.failed = failed; @@ -68,26 +70,26 @@ public BatchDTO(String batch_id, String request_id, Long total, Long ongoing, Lo this.failedAmount = failedAmount; this.file = file; this.notes = notes; - this.created_at = created_at; + this.createdAt = created_at; this.status = status; this.modes = modes; this.purpose = purpose; } public String getBatch_id() { - return batch_id; + return batchId; } public void setBatch_id(String batch_id) { - this.batch_id = batch_id; + this.batchId = batch_id; } public String getRequest_id() { - return request_id; + return requestId; } public void setRequest_id(String request_id) { - this.request_id = request_id; + this.requestId = request_id; } public Long getTotal() { @@ -171,11 +173,11 @@ public void setNotes(String notes) { } public String getCreated_at() { - return created_at; + return createdAt; } public void setCreated_at(String created_at) { - this.created_at = created_at; + this.createdAt = created_at; } public String getStatus() { diff --git a/src/main/java/org/mifos/processor/bulk/schema/GSMATransaction.java b/src/main/java/org/mifos/processor/bulk/schema/GSMATransaction.java index 2d70c7a6..4d6af3ea 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/GSMATransaction.java +++ b/src/main/java/org/mifos/processor/bulk/schema/GSMATransaction.java @@ -13,7 +13,6 @@ public class GSMATransaction implements CsvSchema { private String note; - public int getId() { return id; } diff --git a/src/main/java/org/mifos/processor/bulk/schema/PaymentModalityDTO.java b/src/main/java/org/mifos/processor/bulk/schema/PaymentModalityDTO.java index 20d1053c..035e6119 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/PaymentModalityDTO.java +++ b/src/main/java/org/mifos/processor/bulk/schema/PaymentModalityDTO.java @@ -10,6 +10,7 @@ @NoArgsConstructor @AllArgsConstructor public class PaymentModalityDTO { + private String paymentModality; private String financialAddress; private String bankingInstitutionCode; diff --git a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java index e1c37d0f..e49eb3e5 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java +++ b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java @@ -10,7 +10,8 @@ @Getter @Setter @JsonIgnoreProperties(ignoreUnknown = true) -@JsonPropertyOrder({ "id", "request_id", "payment_mode", "account_number", "payer_identifier_type", "payer_identifier", "payee_identifier_type", "payee_identifier", "amount", "currency", "note", "program_shortcode", "cycle" }) +@JsonPropertyOrder({ "id", "request_id", "payment_mode", "account_number", "payer_identifier_type", "payer_identifier", + "payee_identifier_type", "payee_identifier", "amount", "currency", "note", "program_shortcode", "cycle" }) public class Transaction implements CsvSchema { @JsonProperty("id") diff --git a/src/main/java/org/mifos/processor/bulk/schema/TransactionResult.java b/src/main/java/org/mifos/processor/bulk/schema/TransactionResult.java index f3c6911b..185f2213 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/TransactionResult.java +++ b/src/main/java/org/mifos/processor/bulk/schema/TransactionResult.java @@ -10,9 +10,8 @@ @Getter @Setter @JsonIgnoreProperties(ignoreUnknown = true) -@JsonPropertyOrder({ "id", "request_id", "payment_mode", "payer_identifier_type", "payer_identifier", - "payee_identifier_type", "payee_identifier", "amount", "currency", "note", "program_shortcode", "cycle", - "status", "error_code", "error_description"}) +@JsonPropertyOrder({ "id", "request_id", "payment_mode", "payer_identifier_type", "payer_identifier", "payee_identifier_type", + "payee_identifier", "amount", "currency", "note", "program_shortcode", "cycle", "status", "error_code", "error_description" }) public class TransactionResult extends Transaction { @JsonProperty("status") diff --git a/src/main/java/org/mifos/processor/bulk/utility/PhaseUtils.java b/src/main/java/org/mifos/processor/bulk/utility/PhaseUtils.java index 4b24a00f..b281f8a9 100644 --- a/src/main/java/org/mifos/processor/bulk/utility/PhaseUtils.java +++ b/src/main/java/org/mifos/processor/bulk/utility/PhaseUtils.java @@ -1,11 +1,9 @@ package org.mifos.processor.bulk.utility; - +import java.util.List; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; -import java.util.List; - @Component @ConfigurationProperties(prefix = "callback-phases") public class PhaseUtils { @@ -21,5 +19,4 @@ public void setValues(List values) { } - } diff --git a/src/main/java/org/mifos/processor/bulk/utility/Utils.java b/src/main/java/org/mifos/processor/bulk/utility/Utils.java index 0a4d7ef5..b9cc5c4e 100644 --- a/src/main/java/org/mifos/processor/bulk/utility/Utils.java +++ b/src/main/java/org/mifos/processor/bulk/utility/Utils.java @@ -1,14 +1,25 @@ package org.mifos.processor.bulk.utility; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; import org.mifos.connector.common.channel.dto.TransactionChannelRequestDTO; -import org.mifos.connector.common.gsma.dto.*; +import org.mifos.connector.common.gsma.dto.Fee; +import org.mifos.connector.common.gsma.dto.GSMATransaction; +import org.mifos.connector.common.gsma.dto.GsmaParty; +import org.mifos.connector.common.gsma.dto.IdDocument; +import org.mifos.connector.common.gsma.dto.InternationalTransferInformation; +import org.mifos.connector.common.gsma.dto.Kyc; +import org.mifos.connector.common.gsma.dto.PostalAddress; +import org.mifos.connector.common.gsma.dto.SubjectName; import org.mifos.connector.common.mojaloop.dto.MoneyData; import org.mifos.connector.common.mojaloop.dto.Party; import org.mifos.connector.common.mojaloop.dto.PartyIdInfo; import org.mifos.connector.common.mojaloop.type.IdentifierType; import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.schema.TransactionResult; -import java.io.*; public class Utils { @@ -16,8 +27,7 @@ public static String getTenantSpecificWorkflowId(String originalWorkflowName, St return originalWorkflowName.replace("{dfspid}", tenantName); } - public static String getBulkConnectorBpmnName(String originalWorkflowName, - String paymentMode, String tenantName) { + public static String getBulkConnectorBpmnName(String originalWorkflowName, String paymentMode, String tenantName) { return originalWorkflowName.replace("{MODE}", paymentMode.toLowerCase()).replace("{dfspid}", tenantName); } @@ -35,7 +45,7 @@ public static String mergeCsvFile(String file1, String file2) { isFirstLine = false; continue; } - out.write(str+"\n"); + out.write(str + "\n"); } in.close(); out.close(); @@ -52,18 +62,18 @@ public static String getAwsFileUrl(String baseUrl, String filename) { } /** - * takes initial timer in the ISO 8601 durations format - * for more info check + * takes initial timer in the ISO 8601 durations format for more info check * https://docs.camunda.io/docs/0.26/reference/bpmn-workflows/timer-events/#time-duration * - * @param initialTimer initial timer in the ISO 8601 durations format, ex: PT45S + * @param initialTimer + * initial timer in the ISO 8601 durations format, ex: PT45S * @return next timer value in the ISO 8601 durations format */ - public static String getNextTimer(String initialTimer){ + public static String getNextTimer(String initialTimer) { String stringSecondsValue = initialTimer.split("T")[1].split("S")[0]; int initialSeconds = Integer.parseInt(stringSecondsValue); - int currentPower = (int) ( Math.log(initialSeconds) / Math.log(2) ); + int currentPower = (int) (Math.log(initialSeconds) / Math.log(2)); int next = (int) Math.pow(2, ++currentPower); return String.format("PT%sS", next); @@ -99,7 +109,7 @@ public static GSMATransaction convertTxnToGSMA(Transaction transaction) { gsmaTransaction.setAmount(transaction.getAmount()); gsmaTransaction.setCurrency(transaction.getCurrency()); GsmaParty payer = new GsmaParty(); - //logger.info("Payer {} {}", transaction.getPayerIdentifier(),payer[0].); + // logger.info("Payer {} {}", transaction.getPayerIdentifier(),payer[0].); payer.setKey("msisdn"); payer.setValue(transaction.getPayerIdentifier()); GsmaParty payee = new GsmaParty(); @@ -122,8 +132,7 @@ public static GSMATransaction convertTxnToGSMA(Transaction transaction) { fee[0] = fees; gsmaTransaction.setFees(fee); gsmaTransaction.setGeoCode("37.423825,-122.082900"); - InternationalTransferInformation internationalTransferInformation = - new InternationalTransferInformation(); + InternationalTransferInformation internationalTransferInformation = new InternationalTransferInformation(); internationalTransferInformation.setQuotationReference("string"); internationalTransferInformation.setQuoteId("string"); internationalTransferInformation.setDeliveryMethod("directtoaccount"); @@ -186,17 +195,19 @@ public static GSMATransaction convertTxnToGSMA(Transaction transaction) { gsmaTransaction.setServicingIdentity("string"); gsmaTransaction.setRequestDate("2022-09-28T12:51:19.260+00:00"); - return gsmaTransaction; } public static TransactionChannelRequestDTO convertTxnToInboundTransferPayload(Transaction transaction) { TransactionChannelRequestDTO requestDTO = new TransactionChannelRequestDTO(); - requestDTO.setAmount(new MoneyData(){{ - setCurrency(transaction.getCurrency()); - setAmount(transaction.getAmount()); - }}); + requestDTO.setAmount(new MoneyData() { + + { + setCurrency(transaction.getCurrency()); + setAmount(transaction.getAmount()); + } + }); IdentifierType identifierType; try { diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeClientConfiguration.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeClientConfiguration.java index 6c9007bc..5afd9130 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeClientConfiguration.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeClientConfiguration.java @@ -1,12 +1,11 @@ package org.mifos.processor.bulk.zeebe; import io.camunda.zeebe.client.ZeebeClient; +import java.time.Duration; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import java.time.Duration; - @Configuration public class ZeebeClientConfiguration { @@ -18,12 +17,8 @@ public class ZeebeClientConfiguration { @Bean public ZeebeClient setup() { - return ZeebeClient.newClientBuilder() - .gatewayAddress(zeebeBrokerContactpoint) - .usePlaintext() - .defaultJobPollInterval(Duration.ofMillis(1)) - .defaultJobWorkerMaxJobsActive(2000) - .numJobWorkerExecutionThreads(zeebeClientMaxThreads) - .build(); + return ZeebeClient.newClientBuilder().gatewayAddress(zeebeBrokerContactpoint).usePlaintext() + .defaultJobPollInterval(Duration.ofMillis(1)).defaultJobWorkerMaxJobsActive(2000) + .numJobWorkerExecutionThreads(zeebeClientMaxThreads).build(); } } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeProcessStarter.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeProcessStarter.java index 5f11ab4f..a62e69ea 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeProcessStarter.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeProcessStarter.java @@ -2,18 +2,16 @@ import io.camunda.zeebe.client.ZeebeClient; import io.camunda.zeebe.client.api.response.ProcessInstanceEvent; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; import org.apache.camel.Exchange; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.time.Instant; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - - @Component public class ZeebeProcessStarter { @@ -46,18 +44,13 @@ public void startZeebeWorkflow(String workflowId, Map extraVaria Map variables = new HashMap<>(); variables.put(ZeebeVariables.ORIGIN_DATE, Instant.now().toEpochMilli()); - if(extraVariables != null) { + if (extraVariables != null) { variables.putAll(extraVariables); } // TODO if successful transfer response arrives in X timeout return it otherwise do callback - ProcessInstanceEvent join = zeebeClient.newCreateInstanceCommand() - .bpmnProcessId(workflowId) - .latestVersion() - .variables(variables) - .send() - .join(); - + ProcessInstanceEvent join = zeebeClient.newCreateInstanceCommand().bpmnProcessId(workflowId).latestVersion().variables(variables) + .send().join(); logger.info("zeebee workflow instance from process {}", workflowId); } @@ -69,18 +62,13 @@ public String startZeebeWorkflow(String workflowId, String request, Map { - logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); - - Map variables = job.getVariablesAsMap(); - String batchId = (String) variables.get(BATCH_ID); - String fileName = (String) variables.get("fileName"); - - // TODO: How to get sender information? Hard coded in Channel connector? - byte[] csvFile = fileTransferService.downloadFile(fileName, bucketName); - - CsvSchema schema = CsvSchema.emptySchema().withHeader(); - MappingIterator readValues = csvMapper.readerWithSchemaFor(Transaction.class).with(schema).readValues(csvFile); - - /*while (readValues.hasNext()) { - Transaction current = readValues.next(); - current.setBatchId(batchId); - if (current.getPayment_mode().equals("gsma")) - kafkaTemplate.send(gsmaTopicName, objectMapper.writeValueAsString(current)); - else if (current.getPayment_mode().equals("sclb")) - kafkaTemplate.send(slcbTopicName, objectMapper.writeValueAsString(current)); - }*/ - - client.newCompleteCommand(job.getKey()) - .send(); - }) - .name("bulk-processor") - .maxJobsActive(workerMaxJobs) - .open(); + private void workerBulkProcessor() { + zeebeClient.newWorker().jobType("bulk-processor").handler((client, job) -> { + logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); + + Map variables = job.getVariablesAsMap(); + String batchId = (String) variables.get(BATCH_ID); + String fileName = (String) variables.get("fileName"); + + // TODO: How to get sender information? Hard coded in Channel connector? + byte[] csvFile = fileTransferService.downloadFile(fileName, bucketName); + + CsvSchema schema = CsvSchema.emptySchema().withHeader(); + MappingIterator readValues = csvMapper.readerWithSchemaFor(Transaction.class).with(schema).readValues(csvFile); + + /* + * while (readValues.hasNext()) { Transaction current = readValues.next(); current.setBatchId(batchId); if + * (current.getPayment_mode().equals("gsma")) kafkaTemplate.send(gsmaTopicName, + * objectMapper.writeValueAsString(current)); else if (current.getPayment_mode().equals("sclb")) + * kafkaTemplate.send(slcbTopicName, objectMapper.writeValueAsString(current)); } + */ + + client.newCompleteCommand(job.getKey()).send(); + }).name("bulk-processor").maxJobsActive(workerMaxJobs).open(); } - private void workerCheckTransactions(){ + private void workerCheckTransactions() { String jobType = "check-transactions"; - zeebeClient.newWorker() - .jobType(jobType) - .handler((client, job) -> { - logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); - - Map variables = job.getVariablesAsMap(); - String batchId = (String) variables.get(BATCH_ID); - - Exchange exchange = new DefaultExchange(camelContext); - exchange.setProperty(BATCH_ID, batchId); - producerTemplate.send("direct:check-transactions", exchange); - - client.newCompleteCommand(job.getKey()) - .send(); - }) - .name(jobType) - .maxJobsActive(workerMaxJobs) - .open(); + zeebeClient.newWorker().jobType(jobType).handler((client, job) -> { + logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); + + Map variables = job.getVariablesAsMap(); + String batchId = (String) variables.get(BATCH_ID); + + Exchange exchange = new DefaultExchange(camelContext); + exchange.setProperty(BATCH_ID, batchId); + producerTemplate.send("direct:check-transactions", exchange); + + client.newCompleteCommand(job.getKey()).send(); + }).name(jobType).maxJobsActive(workerMaxJobs).open(); } - private void workerSampleTransactions(){ + private void workerSampleTransactions() { String jobType = "sample-transactions"; - zeebeClient.newWorker() - .jobType(jobType) - .handler((client, job) -> { - logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); - - Map variables = job.getVariablesAsMap(); - String batchId = (String) variables.get(BATCH_ID); - - Exchange exchange = new DefaultExchange(camelContext); - exchange.setProperty(BATCH_ID, batchId); - exchange.setProperty(IS_BATCH_READY, variables.get(IS_SAMPLE_READY)); - producerTemplate.send("direct:sample-transactions", exchange); - - client.newCompleteCommand(job.getKey()) - .send(); - }) - .name(jobType) - .maxJobsActive(workerMaxJobs) - .open(); + zeebeClient.newWorker().jobType(jobType).handler((client, job) -> { + logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); + + Map variables = job.getVariablesAsMap(); + String batchId = (String) variables.get(BATCH_ID); + + Exchange exchange = new DefaultExchange(camelContext); + exchange.setProperty(BATCH_ID, batchId); + exchange.setProperty(IS_BATCH_READY, variables.get(IS_SAMPLE_READY)); + producerTemplate.send("direct:sample-transactions", exchange); + + client.newCompleteCommand(job.getKey()).send(); + }).name(jobType).maxJobsActive(workerMaxJobs).open(); } } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java index b9db7e9e..db31b70a 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java @@ -1,7 +1,13 @@ package org.mifos.processor.bulk.zeebe.worker; +import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_PARTY_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CHANNEL_REQUEST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORIGIN_CHANNEL_REQUEST; +import static org.mifos.processor.bulk.zeebe.worker.Worker.ACCOUNT_LOOKUP_CALLBACK; + import com.fasterxml.jackson.databind.ObjectMapper; import io.camunda.zeebe.client.ZeebeClient; +import java.util.Map; import org.apache.camel.CamelContext; import org.mifos.connector.common.channel.dto.TransactionChannelRequestDTO; import org.mifos.connector.common.mojaloop.dto.Party; @@ -9,43 +15,31 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.util.Map; - -import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_PARTY_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CHANNEL_REQUEST; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORIGIN_CHANNEL_REQUEST; -import static org.mifos.processor.bulk.zeebe.worker.Worker.ACCOUNT_LOOKUP_CALLBACK; - @Component -public class AccountLookupCallbackWorker extends BaseWorker{ +public class AccountLookupCallbackWorker extends BaseWorker { + @Autowired private ZeebeClient zeebeClient; @Autowired private CamelContext camelContext; @Autowired private ObjectMapper objectMapper; + @Override - public void setup(){ + public void setup() { logger.info("## generating " + ACCOUNT_LOOKUP_CALLBACK + "zeebe worker"); - zeebeClient.newWorker() - .jobType(ACCOUNT_LOOKUP_CALLBACK.getValue()) - .handler((client, job) -> { - logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); - Map existingVariables = job.getVariablesAsMap(); - existingVariables.put(ORIGIN_CHANNEL_REQUEST,existingVariables.get(CHANNEL_REQUEST)); - TransactionChannelRequestDTO transactionChannelRequestDTO = objectMapper.readValue((String) existingVariables.get(CHANNEL_REQUEST), TransactionChannelRequestDTO.class); - String payeeId = existingVariables.get(PAYEE_PARTY_ID).toString(); - PartyIdInfo partyIdInfo = new PartyIdInfo(transactionChannelRequestDTO.getPayee().getPartyIdInfo().getPartyIdType(), payeeId); - Party payee =new Party(partyIdInfo); - transactionChannelRequestDTO.setPayee(payee); - existingVariables.put(CHANNEL_REQUEST, objectMapper.writeValueAsString(transactionChannelRequestDTO)); - client.newCompleteCommand(job.getKey()) - .variables(existingVariables) - .send() - .join() - ; - }) - .name(ACCOUNT_LOOKUP_CALLBACK.getValue()) - .open(); + zeebeClient.newWorker().jobType(ACCOUNT_LOOKUP_CALLBACK.getValue()).handler((client, job) -> { + logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); + Map existingVariables = job.getVariablesAsMap(); + existingVariables.put(ORIGIN_CHANNEL_REQUEST, existingVariables.get(CHANNEL_REQUEST)); + TransactionChannelRequestDTO transactionChannelRequestDTO = objectMapper + .readValue((String) existingVariables.get(CHANNEL_REQUEST), TransactionChannelRequestDTO.class); + String payeeId = existingVariables.get(PAYEE_PARTY_ID).toString(); + PartyIdInfo partyIdInfo = new PartyIdInfo(transactionChannelRequestDTO.getPayee().getPartyIdInfo().getPartyIdType(), payeeId); + Party payee = new Party(partyIdInfo); + transactionChannelRequestDTO.setPayee(payee); + existingVariables.put(CHANNEL_REQUEST, objectMapper.writeValueAsString(transactionChannelRequestDTO)); + client.newCompleteCommand(job.getKey()).variables(existingVariables).send().join(); + }).name(ACCOUNT_LOOKUP_CALLBACK.getValue()).open(); } } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java index b0e12a5c..51e84b6e 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java @@ -1,29 +1,35 @@ package org.mifos.processor.bulk.zeebe.worker; -import com.amazonaws.services.dynamodbv2.xspec.S; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.CACHED_TRANSACTION_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HOST; +import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_IDENTITY; +import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYMENT_MODALITY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ACCOUNT_LOOKUP_RETRY_COUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CHANNEL_REQUEST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INITIATOR_FSP_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.IS_RTP_REQUEST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORIGIN_DATE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TRANSACTION_ID; +import static org.mifos.processor.bulk.zeebe.worker.Worker.ACCOUNT_LOOKUP; + import com.fasterxml.jackson.databind.ObjectMapper; import io.camunda.zeebe.client.ZeebeClient; +import java.util.Map; import org.apache.camel.CamelContext; import org.apache.camel.Exchange; import org.apache.camel.ProducerTemplate; import org.apache.camel.support.DefaultExchange; import org.mifos.connector.common.channel.dto.TransactionChannelRequestDTO; -import org.mifos.connector.common.mojaloop.dto.Party; import org.mifos.connector.common.mojaloop.dto.PartyIdInfo; -import org.mifos.connector.common.mojaloop.type.IdentifierType; -import org.mifos.processor.bulk.camel.config.CamelProperties; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import java.util.Map; - -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; -import static org.mifos.processor.bulk.zeebe.worker.Worker.ACCOUNT_LOOKUP; -import static org.mifos.processor.bulk.zeebe.worker.Worker.ACCOUNT_LOOKUP_CALLBACK; - @Component -public class AccountLookupWorker extends BaseWorker{ +public class AccountLookupWorker extends BaseWorker { + @Autowired private ZeebeClient zeebeClient; @@ -37,49 +43,41 @@ public class AccountLookupWorker extends BaseWorker{ private String identityMapperURL; @Value("${bulk-processor.hostname}") private String bulkURL; + @Override public void setup() { logger.info("## generating " + ACCOUNT_LOOKUP + "zeebe worker"); - zeebeClient.newWorker() - .jobType(ACCOUNT_LOOKUP.getValue()) - .handler((client, job) -> { - logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); - Map existingVariables = job.getVariablesAsMap(); - existingVariables.put(ACCOUNT_LOOKUP_RETRY_COUNT, 1); - existingVariables.put(CACHED_TRANSACTION_ID, job.getKey()); - - boolean isTransactionRequest = (boolean) existingVariables.get(IS_RTP_REQUEST); - String tenantId = (String) existingVariables.get(TENANT_ID); - Object channelRequest = existingVariables.get(CHANNEL_REQUEST); - TransactionChannelRequestDTO request = objectMapper.readValue((String) channelRequest, TransactionChannelRequestDTO.class); - - existingVariables.put(INITIATOR_FSP_ID, tenantId); - PartyIdInfo requestedParty = isTransactionRequest ? request.getPayer().getPartyIdInfo() : request.getPayee().getPartyIdInfo(); + zeebeClient.newWorker().jobType(ACCOUNT_LOOKUP.getValue()).handler((client, job) -> { + logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); + Map existingVariables = job.getVariablesAsMap(); + existingVariables.put(ACCOUNT_LOOKUP_RETRY_COUNT, 1); + existingVariables.put(CACHED_TRANSACTION_ID, job.getKey()); - String payeeIdentity = requestedParty.getPartyIdentifier(); - String paymentModality = requestedParty.getPartyIdType().toString(); + boolean isTransactionRequest = (boolean) existingVariables.get(IS_RTP_REQUEST); + String tenantId = (String) existingVariables.get(TENANT_ID); + Object channelRequest = existingVariables.get(CHANNEL_REQUEST); + TransactionChannelRequestDTO request = objectMapper.readValue((String) channelRequest, TransactionChannelRequestDTO.class); + existingVariables.put(INITIATOR_FSP_ID, tenantId); + PartyIdInfo requestedParty = isTransactionRequest ? request.getPayer().getPartyIdInfo() : request.getPayee().getPartyIdInfo(); - Exchange exchange = new DefaultExchange(camelContext); - exchange.setProperty(HOST, identityMapperURL); - exchange.setProperty(PAYEE_IDENTITY, payeeIdentity); - exchange.setProperty(PAYMENT_MODALITY, paymentModality); - exchange.setProperty(CALLBACK,bulkURL + "/accountLookup/Callback"); - exchange.setProperty(TRANSACTION_ID, existingVariables.get(TRANSACTION_ID)); - exchange.setProperty("requestId", job.getKey()); - exchange.setProperty(CHANNEL_REQUEST, channelRequest); - exchange.setProperty(ORIGIN_DATE, existingVariables.get(ORIGIN_DATE)); - exchange.setProperty(TENANT_ID, tenantId); - producerTemplate.send("direct:send-account-lookup", exchange); + String payeeIdentity = requestedParty.getPartyIdentifier(); + String paymentModality = requestedParty.getPartyIdType().toString(); + Exchange exchange = new DefaultExchange(camelContext); + exchange.setProperty(HOST, identityMapperURL); + exchange.setProperty(PAYEE_IDENTITY, payeeIdentity); + exchange.setProperty(PAYMENT_MODALITY, paymentModality); + exchange.setProperty(CALLBACK, bulkURL + "/accountLookup/Callback"); + exchange.setProperty(TRANSACTION_ID, existingVariables.get(TRANSACTION_ID)); + exchange.setProperty("requestId", job.getKey()); + exchange.setProperty(CHANNEL_REQUEST, channelRequest); + exchange.setProperty(ORIGIN_DATE, existingVariables.get(ORIGIN_DATE)); + exchange.setProperty(TENANT_ID, tenantId); + producerTemplate.send("direct:send-account-lookup", exchange); - client.newCompleteCommand(job.getKey()) - .variables(existingVariables) - .send() - ; - }) - .name(String.valueOf(ACCOUNT_LOOKUP)) - .open(); + client.newCompleteCommand(job.getKey()).variables(existingVariables).send(); + }).name(String.valueOf(ACCOUNT_LOOKUP)).open(); } } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/ApprovalWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/ApprovalWorker.java index 36585159..302197cf 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/ApprovalWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/ApprovalWorker.java @@ -1,9 +1,9 @@ package org.mifos.processor.bulk.zeebe.worker; -import org.springframework.stereotype.Component; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.APPROVAL_FAILED; import java.util.Map; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.APPROVAL_FAILED; +import org.springframework.stereotype.Component; @Component public class ApprovalWorker extends BaseWorker { diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BaseWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BaseWorker.java index 8f35ebd1..8262561f 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BaseWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BaseWorker.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.dataformat.csv.CsvMapper; import io.camunda.zeebe.client.ZeebeClient; import io.camunda.zeebe.client.api.worker.JobHandler; +import javax.annotation.PostConstruct; import org.apache.camel.CamelContext; import org.apache.camel.Exchange; import org.apache.camel.ProducerTemplate; @@ -14,8 +15,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; - @Component public abstract class BaseWorker { @@ -49,12 +48,7 @@ public abstract class BaseWorker { public abstract void setup(); public void newWorker(Worker worker, JobHandler handler) { - zeebeClient.newWorker() - .jobType(worker.getValue()) - .handler(handler) - .name(worker.getValue()) - .maxJobsActive(workerMaxJobs) - .open(); + zeebeClient.newWorker().jobType(worker.getValue()).handler(handler).name(worker.getValue()).maxJobsActive(workerMaxJobs).open(); } public void sendToCamelRoute(RouteId routeId, Exchange exchange) { diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java index a1a8196d..663826b4 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java @@ -1,12 +1,18 @@ package org.mifos.processor.bulk.zeebe.worker; +import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_STATUS_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_RATE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_CODE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_DESCRIPTION; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.RETRY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; + +import java.util.Map; import org.apache.camel.Exchange; import org.apache.camel.support.DefaultExchange; import org.mifos.processor.bulk.camel.routes.RouteId; import org.springframework.stereotype.Component; -import java.util.Map; -import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_STATUS_FAILED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @Component public class BatchStatusWorker extends BaseWorker { diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/FormattingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/FormattingWorker.java index cde6a102..34c40d58 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/FormattingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/FormattingWorker.java @@ -1,14 +1,16 @@ package org.mifos.processor.bulk.zeebe.worker; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FORMATTING_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FORMATTING_STANDARD; + +import java.util.Map; import org.apache.camel.Exchange; import org.apache.camel.support.DefaultExchange; import org.mifos.processor.bulk.camel.routes.RouteId; import org.springframework.stereotype.Component; -import java.util.Map; -import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; - @Component public class FormattingWorker extends BaseWorker { @@ -16,11 +18,9 @@ public class FormattingWorker extends BaseWorker { public void setup() { /** - * Starts the new worker for formatting of the data. Performs below tasks - * 1. Downloads the file from cloud. - * 2. Parse the data into POJO. - * 3. Format the data based on field configured in application.yaml - * 4. Uploads the updated file in cloud + * Starts the new worker for formatting of the data. Performs below tasks 1. Downloads the file from cloud. 2. + * Parse the data into POJO. 3. Format the data based on field configured in application.yaml 4. Uploads the + * updated file in cloud */ newWorker(Worker.FORMATTING, (client, job) -> { logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java index c1c75294..787a8596 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java @@ -1,16 +1,26 @@ package org.mifos.processor.bulk.zeebe.worker; -import org.apache.camel.Exchange; -import org.apache.camel.support.DefaultExchange; -import org.mifos.processor.bulk.camel.routes.RouteId; -import org.springframework.stereotype.Component; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_FAILURE_SUB_BATCHES; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_SUB_BATCH_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_SUCCESS_SUB_BATCHES; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REMAINING_SUB_BATCH; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SUB_BATCHES; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; import java.util.ArrayList; import java.util.List; import java.util.Map; -import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; -import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; +import org.apache.camel.Exchange; +import org.apache.camel.support.DefaultExchange; +import org.mifos.processor.bulk.camel.routes.RouteId; +import org.springframework.stereotype.Component; @Component public class InitSubBatchWorker extends BaseWorker { @@ -19,10 +29,8 @@ public class InitSubBatchWorker extends BaseWorker { public void setup() { /** - * Starts the new worker for initialising sub batches. Performs below tasks - * 1. Downloads the file from cloud. - * 2. Parse the data into POJO. - * 3. Initiates workflow based on the payment_mode + * Starts the new worker for initialising sub batches. Performs below tasks 1. Downloads the file from cloud. 2. + * Parse the data into POJO. 3. Initiates workflow based on the payment_mode */ newWorker(Worker.INIT_SUB_BATCH, (client, job) -> { logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java index cf74564c..fd55824f 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java @@ -1,15 +1,22 @@ package org.mifos.processor.bulk.zeebe.worker; -import org.apache.camel.Exchange; -import org.apache.camel.support.DefaultExchange; -import org.mifos.processor.bulk.camel.routes.RouteId; -import org.springframework.stereotype.Component; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_FAILURE_SUB_BATCHES; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_SUCCESS_SUB_BATCHES; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_COMPLETED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_FILE_LIST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_ITERATION; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.RESULT_FILE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SUB_BATCHES; import java.util.ArrayList; import java.util.List; import java.util.Map; - -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; +import org.apache.camel.Exchange; +import org.apache.camel.support.DefaultExchange; +import org.mifos.processor.bulk.camel.routes.RouteId; +import org.springframework.stereotype.Component; @Component public class MergeBackWorker extends BaseWorker { @@ -67,7 +74,6 @@ public void setup() { variables.put(MERGE_COMPLETED, mergeCompleted); variables.put(MERGE_ITERATION, ++mergeIteration); - client.newCompleteCommand(job.getKey()).variables(variables).send(); }); } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java index e4e7e74f..83e41af8 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java @@ -1,21 +1,22 @@ package org.mifos.processor.bulk.zeebe.worker; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERED_BY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERING_FAILED; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; import org.apache.camel.Exchange; import org.apache.camel.support.DefaultExchange; import org.mifos.processor.bulk.camel.routes.RouteId; import org.mifos.processor.bulk.schema.Transaction; import org.springframework.stereotype.Component; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.HashSet; -import java.util.Objects; - -import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; -import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; - @Component public class OrderingWorker extends BaseWorker { @@ -23,11 +24,9 @@ public class OrderingWorker extends BaseWorker { public void setup() { /** - * This worker is responsible for ordering the data set based on field configuration. Performs below tasks. - * 1. Downloads the file from cloud. - * 2. Parse the data into POJO. - * 3. Re-order the data based on field configured in application.yaml - * 4. Uploads the updated file in cloud + * This worker is responsible for ordering the data set based on field configuration. Performs below tasks. 1. + * Downloads the file from cloud. 2. Parse the data into POJO. 3. Re-order the data based on field configured in + * application.yaml 4. Uploads the updated file in cloud */ newWorker(Worker.ORDERING, (client, job) -> { logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); @@ -49,7 +48,7 @@ public void setup() { variables.put(ORDERED_BY, exchange.getProperty(ORDERED_BY)); } - if(workerConfig.isTransactionDeduplicationEnabled){ + if (workerConfig.isTransactionDeduplicationEnabled) { List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); removeDuplicates(transactionList, workerConfig.isOrderingWorkerEnabled); variables.put(TRANSACTION_LIST, transactionList); @@ -58,17 +57,17 @@ public void setup() { }); } - private void removeDuplicates(List transactionList, boolean orderingEnabled){ - if(orderingEnabled){ + private void removeDuplicates(List transactionList, boolean orderingEnabled) { + if (orderingEnabled) { removeDuplicatesIfOrderingEnabled(transactionList); return; } removeDuplicatesIfOrderingDisabled(transactionList); } - private void removeDuplicatesIfOrderingEnabled(List transactionList){ + private void removeDuplicatesIfOrderingEnabled(List transactionList) { - for (int i = 0; i transactionList } } - private void removeDuplicatesIfOrderingDisabled(List transactionList){ + private void removeDuplicatesIfOrderingDisabled(List transactionList) { Set set = new HashSet<>(); - if(Objects.isNull(transactionList)){ + if (Objects.isNull(transactionList)) { return; } - for(Transaction transaction : transactionList){ + for (Transaction transaction : transactionList) { String payeeDetail = fetchPayeeDetail(transaction); - if(set.contains(payeeDetail)){ + if (set.contains(payeeDetail)) { transaction.setNote("Duplicate transaction."); - } - else{ + } else { set.add(payeeDetail); } } } - private String fetchPayeeDetail(Transaction transaction){ + private String fetchPayeeDetail(Transaction transaction) { String payeeIdentifier = transaction.getPayeeIdentifier(); String payeeIdentifierType = transaction.getPayeeIdentifierType(); String amount = transaction.getAmount(); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/PartyLookupWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/PartyLookupWorker.java index 9dc01ec0..0b8a0301 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/PartyLookupWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/PartyLookupWorker.java @@ -1,11 +1,9 @@ package org.mifos.processor.bulk.zeebe.worker; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_FAILED; import java.util.Map; - -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_FAILED; +import org.springframework.stereotype.Component; @Component public class PartyLookupWorker extends BaseWorker { diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java index 1fc0c912..cb362b06 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java @@ -1,16 +1,23 @@ package org.mifos.processor.bulk.zeebe.worker; +import static org.mifos.processor.bulk.camel.config.CamelProperties.CALLBACK_RESPONSE_CODE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_RETRY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_SUCCESS; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_URL; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_RATE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_CODE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_DESCRIPTION; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_CALLBACK_RETRY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_STATUS_RETRY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASES; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASE_COUNT; + +import java.util.Map; import org.apache.camel.Exchange; import org.apache.camel.support.DefaultExchange; import org.mifos.processor.bulk.camel.routes.RouteId; import org.springframework.stereotype.Component; -import java.util.Map; - -import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_STATUS_FAILED; -import static org.mifos.processor.bulk.camel.config.CamelProperties.CALLBACK_RESPONSE_CODE; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; - @Component public class SendCallbackWorker extends BaseWorker { @@ -20,15 +27,15 @@ public void setup() { logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); Map variables = job.getVariablesAsMap(); - int retry = variables.getOrDefault(CALLBACK_RETRY, 0).equals(variables.get(MAX_STATUS_RETRY))?0: - (int) variables.getOrDefault(CALLBACK_RETRY, 0); + int retry = variables.getOrDefault(CALLBACK_RETRY, 0).equals(variables.get(MAX_STATUS_RETRY)) ? 0 + : (int) variables.getOrDefault(CALLBACK_RETRY, 0); Exchange exchange = new DefaultExchange(camelContext); - exchange.setProperty(MAX_CALLBACK_RETRY,variables.get(MAX_CALLBACK_RETRY)); - exchange.setProperty(CALLBACK_RETRY,retry); - exchange.setProperty(CALLBACK_URL,variables.get(CALLBACK_URL)); - exchange.setProperty(COMPLETION_RATE,variables.get(COMPLETION_RATE)); - exchange.setProperty(PHASES,variables.get(PHASES)); - exchange.setProperty(PHASE_COUNT,variables.get(PHASE_COUNT)); + exchange.setProperty(MAX_CALLBACK_RETRY, variables.get(MAX_CALLBACK_RETRY)); + exchange.setProperty(CALLBACK_RETRY, retry); + exchange.setProperty(CALLBACK_URL, variables.get(CALLBACK_URL)); + exchange.setProperty(COMPLETION_RATE, variables.get(COMPLETION_RATE)); + exchange.setProperty(PHASES, variables.get(PHASES)); + exchange.setProperty(PHASE_COUNT, variables.get(PHASE_COUNT)); sendToCamelRoute(RouteId.SEND_CALLBACK, exchange); Boolean callbackSuccess = exchange.getProperty(CALLBACK_SUCCESS, Boolean.class); @@ -37,15 +44,16 @@ public void setup() { variables.put(ERROR_DESCRIPTION, exchange.getProperty(ERROR_DESCRIPTION)); logger.info("Error: {}, {}", variables.get(ERROR_CODE), variables.get(ERROR_DESCRIPTION)); } else { - variables.put(CALLBACK_SUCCESS,true); + variables.put(CALLBACK_SUCCESS, true); } - variables.put(CALLBACK_RETRY,exchange.getProperty(CALLBACK_RETRY)); - variables.put(CALLBACK_RESPONSE_CODE,exchange.getProperty(CALLBACK_RESPONSE_CODE)); - variables.put(PHASE_COUNT,exchange.getProperty(PHASE_COUNT)); - variables.put(PHASES,exchange.getProperty(PHASES)); + variables.put(CALLBACK_RETRY, exchange.getProperty(CALLBACK_RETRY)); + variables.put(CALLBACK_RESPONSE_CODE, exchange.getProperty(CALLBACK_RESPONSE_CODE)); + variables.put(PHASE_COUNT, exchange.getProperty(PHASE_COUNT)); + variables.put(PHASES, exchange.getProperty(PHASES)); - logger.debug("Retry: {} and Response Code {}", exchange.getProperty(CALLBACK_RETRY), exchange.getProperty(CALLBACK_RESPONSE_CODE)); + logger.debug("Retry: {} and Response Code {}", exchange.getProperty(CALLBACK_RETRY), + exchange.getProperty(CALLBACK_RESPONSE_CODE)); client.newCompleteCommand(job.getKey()).variables(variables).send(); }); } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java index ab1bfb36..fb0d8c85 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java @@ -1,15 +1,21 @@ package org.mifos.processor.bulk.zeebe.worker; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_SUB_BATCH_FILE_NAME_ARRAY; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_CREATED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_FAILURE_SUB_BATCHES; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_SUCCESS_SUB_BATCHES; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SUB_BATCHES; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import org.apache.camel.Exchange; import org.apache.camel.support.DefaultExchange; import org.mifos.processor.bulk.camel.routes.RouteId; import org.springframework.stereotype.Component; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; @Component public class SplittingWorker extends BaseWorker { @@ -18,11 +24,9 @@ public class SplittingWorker extends BaseWorker { public void setup() { /** - * This worker performs below tasks - * 1. Downloads the original CSV from cloud - * 2. Splits entire CSV into multiple CSV of sub-batches, based on configured sub-batch size. - * 3. Uploads the sub-batch CSVs to cloud - * 4. Sets zeebeVariable [SPLITTING_FAILED, SUB_BATCHES, SUB_BATCH_CREATED] + * This worker performs below tasks 1. Downloads the original CSV from cloud 2. Splits entire CSV into multiple + * CSV of sub-batches, based on configured sub-batch size. 3. Uploads the sub-batch CSVs to cloud 4. Sets + * zeebeVariable [SPLITTING_FAILED, SUB_BATCHES, SUB_BATCH_CREATED] */ newWorker(Worker.SPLITTING, (client, job) -> { logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); @@ -39,13 +43,12 @@ public void setup() { sendToCamelRoute(RouteId.SPLITTING, exchange); assert !exchange.getProperty(SPLITTING_FAILED, Boolean.class); } catch (Exception e) { - variables.put(SPLITTING_FAILED, true); + variables.put(SPLITTING_FAILED, true); } Boolean subBatchCreated = exchange.getProperty(SUB_BATCH_CREATED, Boolean.class); List serverSubBatchFileList = exchange.getProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, List.class); - if (subBatchCreated != null && !subBatchCreated && - serverSubBatchFileList != null && serverSubBatchFileList.isEmpty()) { + if (subBatchCreated != null && !subBatchCreated && serverSubBatchFileList != null && serverSubBatchFileList.isEmpty()) { // if no sub-batches is created, insert the original filename in sub batch array serverSubBatchFileList.add(filename); subBatchCreated = false; diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java index c995e58a..3f2ed26d 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java @@ -2,18 +2,10 @@ public enum Worker { - PARTY_LOOKUP("partyLookup"), - APPROVAL("approval"), - ORDERING("ordering"), - SPLITTING("splitting"), - FORMATTING("formatting"), - BATCH_STATUS("batchStatus"), - SEND_CALLBACK("sendCallback"), - MERGE_BACK("mergeSubBatch"), + PARTY_LOOKUP("partyLookup"), APPROVAL("approval"), ORDERING("ordering"), SPLITTING("splitting"), FORMATTING("formatting"), BATCH_STATUS( + "batchStatus"), SEND_CALLBACK("sendCallback"), MERGE_BACK("mergeSubBatch"), - INIT_SUB_BATCH("initSubBatch"), - ACCOUNT_LOOKUP("accountLookup"), - ACCOUNT_LOOKUP_CALLBACK("accountLookupCallback"); + INIT_SUB_BATCH("initSubBatch"), ACCOUNT_LOOKUP("accountLookup"), ACCOUNT_LOOKUP_CALLBACK("accountLookupCallback"); private final String value; diff --git a/src/test/java/org/mifos/processor/BulkProcessorApplicationTests.java b/src/test/java/org/mifos/processor/BulkProcessorApplicationTests.java index bbdf1795..a9137489 100644 --- a/src/test/java/org/mifos/processor/BulkProcessorApplicationTests.java +++ b/src/test/java/org/mifos/processor/BulkProcessorApplicationTests.java @@ -1,11 +1,10 @@ package org.mifos.processor; +import java.util.UUID; import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; -import java.util.UUID; - @SpringBootTest @ActiveProfiles("test") class BulkProcessorApplicationTests { diff --git a/src/test/java/org/mifos/processor/cucumber/CucumberContext.java b/src/test/java/org/mifos/processor/cucumber/CucumberContext.java index 86e8eecf..dd58ba78 100644 --- a/src/test/java/org/mifos/processor/cucumber/CucumberContext.java +++ b/src/test/java/org/mifos/processor/cucumber/CucumberContext.java @@ -1,5 +1,7 @@ package org.mifos.processor.cucumber; +import static com.google.common.truth.Truth.assertThat; + import io.cucumber.spring.CucumberContextConfiguration; import org.apache.camel.ProducerTemplate; import org.apache.camel.test.spring.junit5.CamelSpringBootTest; @@ -9,8 +11,6 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; -import static com.google.common.truth.Truth.assertThat; - @CucumberContextConfiguration @SpringBootTest @CamelSpringBootTest diff --git a/src/test/java/org/mifos/processor/cucumber/stepdef/ConfigurationTestStepDef.java b/src/test/java/org/mifos/processor/cucumber/stepdef/ConfigurationTestStepDef.java index 3d137c94..c4957f46 100644 --- a/src/test/java/org/mifos/processor/cucumber/stepdef/ConfigurationTestStepDef.java +++ b/src/test/java/org/mifos/processor/cucumber/stepdef/ConfigurationTestStepDef.java @@ -1,15 +1,16 @@ package org.mifos.processor.cucumber.stepdef; +import static com.google.common.truth.Truth.assertThat; + import io.cucumber.java.en.And; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; +import java.util.function.Function; import org.apache.camel.Exchange; import org.mifos.processor.bulk.config.PaymentModeMapping; import org.mifos.processor.bulk.config.PaymentModeType; import org.mifos.processor.bulk.utility.Utils; -import java.util.function.Function; -import static com.google.common.truth.Truth.assertThat; public class ConfigurationTestStepDef extends BaseStepDef { @@ -50,8 +51,7 @@ public void setPaymentMode(String paymentMode) { @Then("I should get the bulk connector bpmn name {string}") public void validateBulkConnectorBpmnName(String bpmnName) { PaymentModeMapping mapping = BaseStepDef.paymentModeMapping; - String generatedBpmnName = Utils.getBulkConnectorBpmnName(mapping.getEndpoint(), - mapping.getId(), BaseStepDef.tenant); + String generatedBpmnName = Utils.getBulkConnectorBpmnName(mapping.getEndpoint(), mapping.getId(), BaseStepDef.tenant); assertThat(bpmnName).isEqualTo(generatedBpmnName); } diff --git a/src/test/java/org/mifos/processor/cucumber/stepdef/InitRouteStepDef.java b/src/test/java/org/mifos/processor/cucumber/stepdef/InitRouteStepDef.java index e0f67818..2c8729af 100644 --- a/src/test/java/org/mifos/processor/cucumber/stepdef/InitRouteStepDef.java +++ b/src/test/java/org/mifos/processor/cucumber/stepdef/InitRouteStepDef.java @@ -1,17 +1,18 @@ package org.mifos.processor.cucumber.stepdef; +import static com.google.common.truth.Truth.assertThat; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_ELEMENT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYMENT_MODE; + import com.fasterxml.jackson.core.JsonProcessingException; import io.cucumber.java.en.And; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; +import java.util.UUID; import org.mifos.connector.common.channel.dto.TransactionChannelRequestDTO; import org.mifos.connector.common.gsma.dto.GSMATransaction; import org.mifos.processor.bulk.schema.Transaction; -import java.util.UUID; -import static com.google.common.truth.Truth.assertThat; -import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_ELEMENT; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYMENT_MODE; public class InitRouteStepDef extends BaseStepDef { @@ -47,17 +48,20 @@ public void exchangeVariableBooleanCheck(String variableKey, String variableValu public void callRuntimePayloadTestRoute(String paymentMode) { exchange = template.send("direct:dynamic-payload-setter", exchange -> { exchange.setProperty(PAYMENT_MODE, paymentMode); - exchange.setProperty(TRANSACTION_LIST_ELEMENT, new Transaction(){{ - setId(0); - setRequestId(UUID.randomUUID().toString()); - setPaymentMode(paymentMode); - setAmount("100"); - setPayerIdentifierType("MSISDN"); - setPayeeIdentifierType("MSISDN"); - setPayerIdentifier("1234567890"); - setPayeeIdentifier("0987654321"); - setCurrency("INR"); - }}); + exchange.setProperty(TRANSACTION_LIST_ELEMENT, new Transaction() { + + { + setId(0); + setRequestId(UUID.randomUUID().toString()); + setPaymentMode(paymentMode); + setAmount("100"); + setPayerIdentifierType("MSISDN"); + setPayeeIdentifierType("MSISDN"); + setPayerIdentifier("1234567890"); + setPayeeIdentifier("0987654321"); + setCurrency("INR"); + } + }); }); } From bf14c51fe503cd463ba905b2d2fdcb955bf9f90e Mon Sep 17 00:00:00 2001 From: Arka Das <127482132+arkadasfynarfin@users.noreply.github.com> Date: Thu, 22 Jun 2023 16:40:03 +0530 Subject: [PATCH 077/156] fix value fetching from yaml file (#74) --- .../org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java index a8eda20e..6fc8c42c 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java @@ -27,7 +27,7 @@ public class WorkerConfig { @Value("${config.completion-threshold-check.enable}") public boolean isCompletionThresholdCheckEnabled; - @Value("config.deduplication.enabled") + @Value("${config.deduplication.enabled}") public boolean isTransactionDeduplicationEnabled; } From 07f5f61a0472d3c1b804f42fbedb25961f5b25a7 Mon Sep 17 00:00:00 2001 From: Dhaval Maniyar <100835289+logoutdhaval@users.noreply.github.com> Date: Mon, 17 Jul 2023 18:51:18 +0530 Subject: [PATCH 078/156] PHEE-347 update Jfrog URLs in all paymenthub repos (#78) --- build.gradle | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 6652a46e..5b285dac 100644 --- a/build.gradle +++ b/build.gradle @@ -18,12 +18,9 @@ repositories { maven { url = uri('https://repo.maven.apache.org/maven2') } - maven { - url "https://fynarfin.jfrog.io/artifactory/fyn-libs-release-local" - } maven { - url = uri('https://fynarfin.jfrog.io/artifactory/fyn-libs-snapshot-local/') + url = uri('https://jfrog.sandbox.fynarfin.io/artifactory/fyn-libs-snapshot') } } From 01c50b30603c5de041e9840f5db3d717d4fbcb1d Mon Sep 17 00:00:00 2001 From: Arka Das <127482132+arkadasfynarfin@users.noreply.github.com> Date: Thu, 20 Jul 2023 20:51:49 +0530 Subject: [PATCH 079/156] PHEE-345: fix NPE when converting Long object to int (#77) --- .../org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java | 2 +- .../mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java index 8e2edeea..d00fc449 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java @@ -52,7 +52,7 @@ public void configure() throws Exception { .unmarshal().json(JsonLibrary.Jackson, BatchDTO.class).process(exchange -> { BatchDTO batchSummary = exchange.getIn().getBody(BatchDTO.class); - long percentage = (long) (((double) batchSummary.getSuccessful() / batchSummary.getTotal()) * 100); + int percentage = (int) (((double) batchSummary.getSuccessful() / batchSummary.getTotal()) * 100); if (percentage >= completionThreshold) { logger.info("Batch success threshold reached. Expected rate: {}, Actual Rate: {}", completionThreshold, percentage); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java index 663826b4..e2edc0c4 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java @@ -34,7 +34,7 @@ public void setup() { Boolean batchStatusFailed = exchange.getProperty(BATCH_STATUS_FAILED, Boolean.class); if (batchStatusFailed == null || !batchStatusFailed) { - successRate = exchange.getProperty(COMPLETION_RATE, Long.class).intValue(); + successRate = exchange.getProperty(COMPLETION_RATE, Integer.class); } else { variables.put(ERROR_CODE, exchange.getProperty(ERROR_CODE)); variables.put(ERROR_DESCRIPTION, exchange.getProperty(ERROR_DESCRIPTION)); From ad07ef04d8e9d3230927d519cab9b2e5ae2d8c2a Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Thu, 20 Jul 2023 20:55:09 +0530 Subject: [PATCH 080/156] Added liveness route manually to be sync with spring actuator endpoint (#80) --- .../bulk/camel/routes/HealthRoute.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/HealthRoute.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/HealthRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/HealthRoute.java new file mode 100644 index 00000000..c9cf0e78 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/HealthRoute.java @@ -0,0 +1,17 @@ +package org.mifos.processor.bulk.camel.routes; + +import org.json.JSONObject; +import org.springframework.stereotype.Component; + +@Component +public class HealthRoute extends BaseRouteBuilder { + + @Override + public void configure() throws Exception { + + // todo remove once camel APIs are migrated to spring + from("rest:GET:/actuator/health/liveness") + .id("rest:GET:/actuator/health/liveness") + .setBody(exchange -> new JSONObject(){{ put("status", "UP"); }}.toString()); + } +} From af40729c7d67bdc3b91c4d4d870a9d065b6faf38 Mon Sep 17 00:00:00 2001 From: Ishan Ranasingh <44061541+truthfool@users.noreply.github.com> Date: Wed, 26 Jul 2023 20:39:09 +0530 Subject: [PATCH 081/156] PHEE-346 Parameterised region (#84) * parameterised region * removed indent --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ed5fcf96..19667102 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -28,7 +28,7 @@ jobs: aws-secret-access-key: AWS_SECRET_ACCESS_KEY extra-build-args: '--compress' push-image: true - region: ap-south-1 + region: "$REGION" registry-id: AWS_REGISTRY_ID repo: phee-bulk-processor repo-scan-on-push: true From b7fd6b0579c20563a809145ab95af8ee26ec378a Mon Sep 17 00:00:00 2001 From: Dhruv Sonagara <78945411+dhruvsonagara@users.noreply.github.com> Date: Mon, 31 Jul 2023 15:07:38 +0530 Subject: [PATCH 082/156] PHEE-374 Update readme for checkstyle and spotless (#87) --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 5e3ec501..8bf4ac15 100644 --- a/README.md +++ b/README.md @@ -1 +1,13 @@ #Auto-Trigger + +## Checkstyle +Use below command to execute the checkstyle test. +```shell +./gradlew checkstyleMain +``` + +## Spotless +Use below command to execute the spotless apply. +```shell +./gradlew spotlessApply +``` From dd130452d27abacfb114bb965450be23f4881105 Mon Sep 17 00:00:00 2001 From: somanath21 <104554892+somanath21@users.noreply.github.com> Date: Tue, 1 Aug 2023 12:54:00 +0530 Subject: [PATCH 083/156] #GOV-527 build and push docker image to Dockerhub (#85) * CircleCI to build push docker image to dockerhub on github tag release * no filter for latest tag * remove imagetag from latest build --------- Co-authored-by: Somanath Hugar --- .circleci/config.yml | 109 +++++++++++++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 36 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 19667102..9fe15694 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,46 +1,83 @@ version: 2.1 -orbs: - slack: circleci/slack@4.12.5 - aws-ecr: circleci/aws-ecr@8.2.1 -jobs: - build: +executors: + docker-executor: docker: - - image: cimg/openjdk:17.0.0 - - image: docker:17.05.0-ce-git - working_directory: ~/repo + - image: circleci/openjdk:17-buster-node-browsers-legacy + +jobs: + build_and_push_tag_image: + executor: docker-executor + environment: + JVM_OPTS: -Xmx512m + TERM: dumb + GITHUB_TOKEN: ${GITHUB_TOKEN} # Add the GitHub token as an environment variable + + steps: + - checkout + - setup_remote_docker: + version: 20.10.14 + - run: + name: Build and Push Docker tag Image + command: | + # Set environment variables + IMAGE_TAG=$CIRCLE_TAG + + # Check if the Docker image with the same tag already exists in Docker Hub + if curl -s -f -u "$DOCKERHUB_USERNAME":"$DOCKERHUB_PASSWORD" "https://hub.docker.com/v2/repositories/openmf/ph-ee-bulk-processor/tags/$IMAGE_TAG" > /dev/null; then + echo "Skipping the build and push as the tag $IMAGE_TAG already exists in Docker Hub." + exit 0 + fi + + # Build and tag the Docker image + ./gradlew bootJar + docker build -t "openmf/ph-ee-bulk-processor:$IMAGE_TAG" . + + # Push the Docker image to Docker Hub + docker login -u "$DOCKERHUB_USERNAME" -p "$DOCKERHUB_PASSWORD" + docker push "openmf/ph-ee-bulk-processor:$IMAGE_TAG" + + # when: always # The job will be executed even if there's no match for the tag filter + + build_and_push_latest_image: + executor: docker-executor environment: - # Customize the JVM maximum heap limit JVM_OPTS: -Xmx512m TERM: dumb + steps: - checkout - - setup_remote_docker -# - slack/notify: -# event: fail -# mentions: '@here' -# template: basic_fail_1 -# - slack/notify: -# event: pass -# template: basic_success_1 - - run: ./gradlew clean bootJar - - aws-ecr/build-and-push-image: - aws-access-key-id: AWS_ACCESS_KEY_ID - aws-secret-access-key: AWS_SECRET_ACCESS_KEY - extra-build-args: '--compress' - push-image: true - region: "$REGION" - registry-id: AWS_REGISTRY_ID - repo: phee-bulk-processor - repo-scan-on-push: true - role-arn: arn:aws:iam::419830066942:role/CustomAdmin - tag: latest - # - run: ./gradlew cucumberCli - # run tests! Slack Success/Fail Notification Step - #- run: ./gradlew test + # Install Docker to build and push the image + - setup_remote_docker: + version: 20.10.14 + + # Build the Docker image + - run: + name: Build Docker image + command: | + ./gradlew bootJar + docker build -t openmf/ph-ee-bulk-processor:latest . + + # Log in to DockerHub using environment variables + - run: + name: Login to DockerHub + command: echo "${DOCKERHUB_PASSWORD}" | docker login -u "${DOCKERHUB_USERNAME}" --password-stdin + + # Push the Docker image to DockerHub + - run: + name: Push Docker image to DockerHub + command: docker push openmf/ph-ee-bulk-processor:latest + workflows: - build_and_push_image: + version: 2 + build-and-push: jobs: - - build: + - build_and_push_tag_image: + filters: + tags: + only: /^v\d+\.\d+\.\d+$/ # Match tags in the format v1.2.3 + context: + - DOCKER + - build_and_push_latest_image: context: - - AWS - - slack + - DOCKER + From c631880409f35d988b3bc54f08be629f8eaddb17 Mon Sep 17 00:00:00 2001 From: Anover000 <140409886+Anover000@users.noreply.github.com> Date: Wed, 9 Aug 2023 23:10:17 +0530 Subject: [PATCH 084/156] Add auth cred for docker-executor (#96) --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9fe15694..ff638dda 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,6 +3,9 @@ executors: docker-executor: docker: - image: circleci/openjdk:17-buster-node-browsers-legacy + auth: + username: $DOCKERHUB_USERNAME + password: $DOCKERHUB_PASSWORD jobs: build_and_push_tag_image: From 733df2bc32405ab1bdfff3d1fc4bcbf3f34cbc0f Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Fri, 18 Aug 2023 16:33:40 +0530 Subject: [PATCH 085/156] GOV 285: Merging tt\he spring migration to master (#100) * Migrated bulk transfer and batch transaction API (#52) * Migrated bulk transfer and batch transaction API * Reverted unnescessary changes * removed csv files * GOV-289.2 Integrated JWS (#53) * Suggested Callback Time that is Resolved by Polling (#51) * Integrated JWS --------- Co-authored-by: Ankita Ranjan <92036361+ankita10r@users.noreply.github.com> * GOV 293: Integrating /batchtransactions with 1.5.0 version of connector common (#58) * GOV-290 Enabled TLS for tomcat server (#60) * GOV 293: Integrating /batchtransactions with 1.5.0 version of connector common * X-Signature header integrated while sending callback * Fixed the version of connector-common * GOV-290 Enabled TLS for tomcat server * Deleted redundant CSV file * Updated readme with info about disabling TLS in tomcat * Added default password for keystore file * Gov 293 X-Signature header integrated while sending callback (#59) * GOV 293: Integrating /batchtransactions with 1.5.0 version of connector common * X-Signature header integrated while sending callback * Fixed the version of connector-common * Deleted redundant CSV file * Migrated bulk transfer and batch transaction API (#52) * Migrated bulk transfer and batch transaction API * Reverted unnescessary changes * removed csv files * GOV-255: Added configuration for programId/registeringInstituteId (#86) * Added configuration for programId/registeringInstituteId * Added logic to support programId and registeringInstituteId header * Updated the config for n cross n mapping b/w registeringInstituteId and program * GOV-255.2 Publishing programName, payerIdentifierType and payerIdentifierValue as zeebe variable * Addressed the comment for import fixes * remove unnecessary merge conflict text from application.yaml (#88) * Code structure fixed * APIOriginFilter bug fixed * Added commons dependency * Spring migration file stream to camel fixed * Added multipart excecption handling * Migrated /simulate api to spring * Address the PR review comments --------- Co-authored-by: Dhruv Sonagara <78945411+dhruvsonagara@users.noreply.github.com> Co-authored-by: Ankita Ranjan <92036361+ankita10r@users.noreply.github.com> Co-authored-by: Arka Das <127482132+arkadasfynarfin@users.noreply.github.com> --- README.md | 13 + build.gradle | 11 +- .../processor/BulkProcessorApplication.java | 16 ++ .../API/definition/BatchTransactions.java | 26 ++ .../bulk/API/definition/BulkTransfer.java | 21 ++ .../bulk/API/definition/Simulate.java | 13 + .../BatchTransactionsController.java | 69 +++++ .../BulkTransferController.java | 45 +++ .../implementation/SimulateApiController.java | 14 + .../processor/bulk/api/ApiOriginFilter.java | 13 +- .../bulk/camel/config/CamelProperties.java | 5 + .../camel/routes/FileProcessingRoute.java | 3 +- .../camel/routes/ProcessorStartRoute.java | 272 ++++++++++++------ .../bulk/camel/routes/SendCallbackRoute.java | 10 +- .../bulk/config/BudgetAccountConfig.java | 24 ++ .../mifos/processor/bulk/config/Program.java | 16 ++ .../config/RegisteringInstitutionConfig.java | 26 ++ .../bulk/file/FileStorageService.java | 11 + .../bulk/file/FileStorageServiceImpl.java | 26 ++ .../processor/bulk/schema/Transaction.java | 5 +- .../mifos/processor/bulk/utility/Headers.java | 44 +++ .../processor/bulk/utility/PhaseUtils.java | 2 +- .../bulk/utility/SpringWrapperUtil.java | 20 ++ .../processor/bulk/zeebe/ZeebeVariables.java | 4 +- src/main/resources/application.yaml | 28 +- src/main/resources/keystore.jks | Bin 0 -> 2272 bytes 26 files changed, 623 insertions(+), 114 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/API/definition/BatchTransactions.java create mode 100644 src/main/java/org/mifos/processor/bulk/API/definition/BulkTransfer.java create mode 100644 src/main/java/org/mifos/processor/bulk/API/definition/Simulate.java create mode 100644 src/main/java/org/mifos/processor/bulk/API/implementation/BatchTransactionsController.java create mode 100644 src/main/java/org/mifos/processor/bulk/API/implementation/BulkTransferController.java create mode 100644 src/main/java/org/mifos/processor/bulk/API/implementation/SimulateApiController.java create mode 100644 src/main/java/org/mifos/processor/bulk/config/BudgetAccountConfig.java create mode 100644 src/main/java/org/mifos/processor/bulk/config/Program.java create mode 100644 src/main/java/org/mifos/processor/bulk/config/RegisteringInstitutionConfig.java create mode 100644 src/main/java/org/mifos/processor/bulk/file/FileStorageService.java create mode 100644 src/main/java/org/mifos/processor/bulk/file/FileStorageServiceImpl.java create mode 100644 src/main/java/org/mifos/processor/bulk/utility/Headers.java create mode 100644 src/main/java/org/mifos/processor/bulk/utility/SpringWrapperUtil.java create mode 100644 src/main/resources/keystore.jks diff --git a/README.md b/README.md index 8bf4ac15..9e3835a6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,18 @@ #Auto-Trigger +## SSL Configuration +```yaml +server: + ssl: + key-alias: "tomcat-https" + key-store: "classpath:keystore.jks" + key-store-type: JKS + key-password: "" + key-store-password: "" + port: 8443 +``` +#### NOTE: For disabling TLS, change the port to "8080" and add null values for all the "ssl" related fields. + ## Checkstyle Use below command to execute the checkstyle test. ```shell diff --git a/build.gradle b/build.gradle index 5b285dac..f91d2dc4 100644 --- a/build.gradle +++ b/build.gradle @@ -107,7 +107,16 @@ dependencies { // miscellaneous dependency implementation 'com.google.code.gson:gson:2.8.9' implementation 'org.json:json:20210307' - implementation 'org.mifos:ph-ee-connector-common:1.4.1-SNAPSHOT' + implementation 'org.mifos:ph-ee-connector-common:1.5.0-SNAPSHOT' + implementation 'org.apache.camel.springboot:camel-spring-boot-starter:3.4.0' + implementation 'org.apache.camel:camel-undertow:3.4.0' + implementation 'org.springframework.boot:spring-boot-starter:2.5.2' + implementation 'org.springframework.boot:spring-boot-starter-web:2.5.2' + implementation 'org.apache.camel:camel-http:3.4.0' + implementation 'org.springframework:spring-web:5.3.19' + implementation 'com.amazonaws:aws-java-sdk:1.11.486' + implementation 'commons-io:commons-io:2.11.0' + implementation 'com.amazonaws:aws-java-sdk-s3:1.11.486' //To be removed implementation 'com.amazonaws:aws-java-sdk-dynamodb:1.11.486' diff --git a/src/main/java/org/mifos/processor/BulkProcessorApplication.java b/src/main/java/org/mifos/processor/BulkProcessorApplication.java index 4602e694..a73fd9cd 100644 --- a/src/main/java/org/mifos/processor/BulkProcessorApplication.java +++ b/src/main/java/org/mifos/processor/BulkProcessorApplication.java @@ -7,12 +7,18 @@ import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import org.apache.camel.Processor; +import org.mifos.connector.common.interceptor.annotation.EnableJsonWebSignature; +import org.mifos.processor.bulk.api.ApiOriginFilter; import org.mifos.processor.bulk.camel.config.HttpClientConfigurerTrustAllCACerts; import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.context.annotation.Bean; @SpringBootApplication +@EnableJsonWebSignature public class BulkProcessorApplication { public static void main(String[] args) { @@ -44,4 +50,14 @@ public HttpClientConfigurerTrustAllCACerts httpClientConfigurer() { return new HttpClientConfigurerTrustAllCACerts(); } + @Bean + public FilterRegistrationBean apiOriginFilter() { + FilterRegistrationBean registration = new FilterRegistrationBean<>(); + registration.setFilter(new ApiOriginFilter()); + registration.addUrlPatterns("/**"); + registration.setName("apiOriginFilter"); + registration.setOrder(Integer.MIN_VALUE+1); + return registration; + } + } diff --git a/src/main/java/org/mifos/processor/bulk/API/definition/BatchTransactions.java b/src/main/java/org/mifos/processor/bulk/API/definition/BatchTransactions.java new file mode 100644 index 00000000..90201f7f --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/API/definition/BatchTransactions.java @@ -0,0 +1,26 @@ +package org.mifos.processor.bulk.api.definition; + +import com.amazonaws.services.dynamodbv2.xspec.S; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.camel.util.json.JsonObject; +import org.json.JSONObject; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import javax.mail.Multipart; +import javax.servlet.http.HttpServletResponse; + +import java.io.IOException; + +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; + +public interface BatchTransactions { + + @PostMapping(value = "/batchtransactions", produces="application/json") + String batchTransactions(HttpServletResponse httpServletResponse, @RequestHeader(value = "X-CorrelationID") String requestId, + @RequestParam("data") MultipartFile file, + @RequestHeader(value = FILE_NAME) String fileName, + @RequestHeader(value = PURPOSE) String purpose, + @RequestHeader(value = "Type") String type, + @RequestHeader(value = "Platform-TenantId") String tenant) throws IOException; +} diff --git a/src/main/java/org/mifos/processor/bulk/API/definition/BulkTransfer.java b/src/main/java/org/mifos/processor/bulk/API/definition/BulkTransfer.java new file mode 100644 index 00000000..328c8c28 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/API/definition/BulkTransfer.java @@ -0,0 +1,21 @@ +package org.mifos.processor.bulk.api.definition; + +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; + +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; + +public interface BulkTransfer { + @PostMapping(value = "/bulk/transfer/{requestId}/{fileName}", produces = "application/json") + String bulkTransfer(@RequestHeader(value = "X-CorrelationID", required = false) String requestId, + @RequestParam("data") MultipartFile file, + @RequestHeader(value = FILE_NAME, required = false) String fileName, + @RequestHeader(value = PURPOSE, required = false) String purpose, + @RequestHeader(value = "Type", required = false) String type, + @RequestHeader(value = "Platform-TenantId") String tenant) throws IOException; +} diff --git a/src/main/java/org/mifos/processor/bulk/API/definition/Simulate.java b/src/main/java/org/mifos/processor/bulk/API/definition/Simulate.java new file mode 100644 index 00000000..81518166 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/API/definition/Simulate.java @@ -0,0 +1,13 @@ +package org.mifos.processor.bulk.api.definition; + +import org.springframework.web.bind.annotation.PostMapping; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +// from("rest:post:/simulate").log("Reached Simulation"); +public interface Simulate { + + @PostMapping(value = "/simulate", produces="application/json") + void simulate(HttpServletResponse httpServletResponse) throws IOException; + +} diff --git a/src/main/java/org/mifos/processor/bulk/API/implementation/BatchTransactionsController.java b/src/main/java/org/mifos/processor/bulk/API/implementation/BatchTransactionsController.java new file mode 100644 index 00000000..338152d2 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/API/implementation/BatchTransactionsController.java @@ -0,0 +1,69 @@ +package org.mifos.processor.bulk.api.implementation; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.camel.Exchange; +import org.apache.camel.ProducerTemplate; +import org.json.JSONObject; +import org.mifos.processor.bulk.api.definition.BatchTransactions; +import org.mifos.processor.bulk.file.FileStorageService; +import org.mifos.processor.bulk.utility.Headers; +import org.mifos.processor.bulk.utility.SpringWrapperUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartException; +import org.springframework.web.multipart.MultipartFile; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; + +@Slf4j +@RestController +public class BatchTransactionsController implements BatchTransactions { + + @Autowired + private ProducerTemplate producerTemplate; + + @Autowired + ObjectMapper objectMapper; + + @Autowired + FileStorageService fileStorageService; + + @SneakyThrows + @Override + public String batchTransactions(HttpServletResponse httpServletResponse, + String requestId, MultipartFile file, String fileName, + String purpose, String type, String tenant) throws IOException { + log.info("Inside api logic"); + String localFileName = fileStorageService.save(file); + Headers headers = new Headers.HeaderBuilder() + .addHeader("X-CorrelationID", requestId) + .addHeader(PURPOSE,purpose) + .addHeader(FILE_NAME,localFileName) + .addHeader("Type",type) + .addHeader("Platform-TenantId",tenant) + .build(); + + Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(), + headers); + exchange = producerTemplate.send("direct:post-batch-transactions", exchange); + int statusCode = exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE, Integer.class); + httpServletResponse.setStatus(statusCode); + return exchange.getIn().getBody(String.class); + } + + @ExceptionHandler({MultipartException.class}) + public String handleMultipartException(HttpServletResponse httpServletResponse) { + JSONObject json = new JSONObject(); + json.put("Error Information: ", "File not uploaded"); + json.put("Error Description : ", "There was no fie uploaded with the request. " + + "Please upload a file and try again."); + httpServletResponse.setStatus(httpServletResponse.SC_BAD_REQUEST); + return json.toString(); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/API/implementation/BulkTransferController.java b/src/main/java/org/mifos/processor/bulk/API/implementation/BulkTransferController.java new file mode 100644 index 00000000..084e5f85 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/API/implementation/BulkTransferController.java @@ -0,0 +1,45 @@ +package org.mifos.processor.bulk.api.implementation; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.camel.Exchange; +import org.apache.camel.ProducerTemplate; +import org.mifos.processor.bulk.api.definition.BulkTransfer; +import org.mifos.processor.bulk.file.FileStorageService; +import org.mifos.processor.bulk.utility.Headers; +import org.mifos.processor.bulk.utility.SpringWrapperUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; + +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; + +@RestController +public class BulkTransferController implements BulkTransfer { + @Autowired + private ProducerTemplate producerTemplate; + + @Autowired + ObjectMapper objectMapper; + + @Autowired + FileStorageService fileStorageService; + + @Override + public String bulkTransfer(String requestId, MultipartFile file, String fileName, String purpose, String type, String tenant) throws IOException { + Headers headers = new Headers.HeaderBuilder() + .addHeader("X-CorrelationID", requestId) + .addHeader(PURPOSE,purpose) + .addHeader(FILE_NAME,fileName) + .addHeader("Type",type) + .addHeader("Platform-TenantId",tenant) + .build(); + Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(), + headers); + fileStorageService.save(file); + producerTemplate.send("direct:post-bulk-transfer", exchange); + return exchange.getIn().getBody(String.class); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/API/implementation/SimulateApiController.java b/src/main/java/org/mifos/processor/bulk/API/implementation/SimulateApiController.java new file mode 100644 index 00000000..1557747c --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/API/implementation/SimulateApiController.java @@ -0,0 +1,14 @@ +package org.mifos.processor.bulk.api.implementation; + +import org.mifos.processor.bulk.api.definition.Simulate; +import org.springframework.web.bind.annotation.RestController; +import javax.servlet.http.HttpServletResponse; + +@RestController +public class SimulateApiController implements Simulate { + + @Override + public void simulate(HttpServletResponse httpServletResponse) { + httpServletResponse.setStatus(HttpServletResponse.SC_OK); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java b/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java index 46d29187..f43be522 100644 --- a/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java +++ b/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java @@ -11,17 +11,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; +import org.springframework.web.filter.GenericFilterBean; -@Component -public class ApiOriginFilter implements Filter { +public class ApiOriginFilter extends GenericFilterBean { private Logger logger = LoggerFactory.getLogger(this.getClass()); - @Override - public void init(FilterConfig filterConfig) throws ServletException { - - } - @Override public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { HttpServletRequest req = (HttpServletRequest) request; @@ -30,8 +25,4 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha logger.info("Client IP Address: {}",req.getRemoteHost()); } - @Override - public void destroy() { - - } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 292e15bc..b13d938c 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -70,5 +70,10 @@ private CamelProperties() {} public static final String PAYMENT_MODALITY = "paymentModality"; public static final String PAYEE_PARTY_ID = "payeePartyId"; public static final String PAYEE_PARTY_ID_TYPE = "payeePartyIdType"; + public static final String HEADER_REGISTERING_INSTITUTE_ID = "X-Registering-Institution-ID"; + public static final String HEADER_PROGRAM_ID = "X-Program-ID"; + public static final String REGISTERING_INSTITUTE_ID = "registeringInstituteId"; + public static final String PROGRAM_ID = "programId"; + public static final String IS_UPDATED = "isUpdated"; } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java index 2080b774..9959bcc4 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java @@ -9,7 +9,6 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FAILED_AMOUNT; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ONGOING_AMOUNT; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TOTAL_AMOUNT; - import com.fasterxml.jackson.databind.MappingIterator; import com.fasterxml.jackson.databind.SequenceWriter; import com.fasterxml.jackson.dataformat.csv.CsvMapper; @@ -43,7 +42,9 @@ public void configure() { Long failedAmount = 0L; Long completedAmount = 0L; String filename = exchange.getProperty(LOCAL_FILE_PATH, String.class); + log.debug("Local file path: {}", filename); CsvSchema schema = CsvSchema.emptySchema().withHeader(); + log.info("Filename: {}", filename); FileReader reader = new FileReader(filename); MappingIterator readValues = csvMapper.readerWithSchemaFor(Transaction.class).with(schema) .readValues(reader); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index df97d7aa..172f65ad 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -1,7 +1,14 @@ package org.mifos.processor.bulk.camel.routes; import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_REQUEST_TYPE; +import static org.mifos.processor.bulk.camel.config.CamelProperties.REGISTERING_INSTITUTE_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.PROGRAM_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.IS_UPDATED; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; +import static org.mifos.processor.bulk.camel.config.CamelProperties.RESULT_TRANSACTION_LIST; +import static org.mifos.processor.bulk.camel.config.CamelProperties.OVERRIDE_HEADER; +import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.APPROVAL_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BULK_NOTIF_FAILURE; @@ -10,12 +17,10 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_THRESHOLD; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_THRESHOLD_CHECK_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_VALIDITY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FORMATTING_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_CALLBACK_RETRY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_STATUS_RETRY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_ENABLED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.NOTE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERING_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASES; @@ -25,29 +30,22 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.THRESHOLD_DELAY; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; -import javax.activation.DataHandler; -import javax.mail.internet.MimeBodyPart; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PROGRAM_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYER_IDENTIFIER_TYPE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYER_IDENTIFIER_VALUE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.NOTE; +import java.io.*; +import java.util.*; +import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; import org.apache.tika.Tika; import org.json.JSONObject; +import org.mifos.processor.bulk.config.BudgetAccountConfig; +import org.mifos.processor.bulk.config.Program; +import org.mifos.processor.bulk.config.RegisteringInstitutionConfig; import org.mifos.processor.bulk.file.FileTransferService; +import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.utility.PhaseUtils; import org.mifos.processor.bulk.utility.Utils; import org.mifos.processor.bulk.zeebe.ZeebeProcessStarter; @@ -58,6 +56,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; @Component public class ProcessorStartRoute extends BaseRouteBuilder { @@ -104,6 +103,9 @@ public class ProcessorStartRoute extends BaseRouteBuilder { @Autowired PhaseUtils phaseUtils; + @Autowired + BudgetAccountConfig budgetAccountConfig; + @Override public void configure() { setup(); @@ -111,44 +113,128 @@ public void configure() { private void setup() { - from("rest:POST:/batchtransactions").id("rest:POST:/batchtransactions").log("Starting route rest:POST:/batchtransactions") - .to("direct:validate-file").choice().when(header("CamelHttpResponseCode").isNotEqualTo("200")) - .log(LoggingLevel.ERROR, "File upload failed").otherwise().unmarshal().mimeMultipart("multipart/*").process(exchange -> { + from("direct:post-batch-transactions") + .id("rest:POST:/batchtransactions") + .log("Starting route rest:POST:/batchtransactions") + .to("direct:validate-file") + .choice() + .when(header("CamelHttpResponseCode").isNotEqualTo("200")) + .log(LoggingLevel.ERROR, "File upload failed") + .otherwise() + .process(exchange -> { String batchId = UUID.randomUUID().toString(); exchange.setProperty(BATCH_ID, batchId); - }).unmarshal().mimeMultipart("multipart/*").to("direct:validateFileSyncResponse").choice() - .when(header("CamelHttpResponseCode").isNotEqualTo("200")).log(LoggingLevel.ERROR, "File upload failed").otherwise() - .unmarshal().mimeMultipart("multipart/*").wireTap("direct:executeBatch").to("direct:pollingOutput").endChoice().endChoice(); + }) + .to("direct:validateFileSyncResponse") + .choice() + .when(header("CamelHttpResponseCode").isNotEqualTo("200")) + .log(LoggingLevel.ERROR, "File upload failed").otherwise() + .wireTap("direct:executeBatch").to("direct:pollingOutput").endChoice().endChoice(); - from("rest:POST:/bulk/transfer/{requestId}/{fileName}").unmarshal().mimeMultipart("multipart/*").to("direct:validate-tenant") + from("direct:post-bulk-transfer") + .unmarshal().mimeMultipart("multipart/*") + .to("direct:validate-tenant") .process(exchange -> { - String fileName = System.currentTimeMillis() + "_" + exchange.getIn().getHeader("fileName", String.class); + String fileName = System.currentTimeMillis() + "_" + exchange.getIn().getHeader("fileName", String.class); String requestId = exchange.getIn().getHeader("requestId", String.class); String purpose = exchange.getIn().getHeader("purpose", String.class); String batchId = UUID.randomUUID().toString(); - exchange.setProperty(BATCH_ID, batchId); + exchange.setProperty(BATCH_ID,batchId); exchange.setProperty(FILE_NAME, fileName); exchange.setProperty(REQUEST_ID, requestId); exchange.setProperty(PURPOSE, purpose); - }).wireTap("direct:start-batch-process-csv").to("direct:pollingOutput"); - - from("direct:validate-tenant").id("direct:validate-tenant").log("Validating tenant").process(exchange -> { - String tenantName = exchange.getIn().getHeader("Platform-TenantId", String.class); - // validation is disabled for now - /* - * if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { throw new - * Exception("Invalid tenant value."); } - */ - exchange.setProperty(TENANT_NAME, tenantName); - }).setHeader("Content-Type", constant("application/json;charset=UTF-8")).log("Completed route direct:validate-tenant"); - - from("direct:start-batch-process-csv").id("direct:start-batch-process-csv").log("Starting route direct:start-batch-process-csv") + }) + .wireTap("direct:start-batch-process-csv") + .to("direct:pollingOutput"); + + from("direct:validate-tenant") + .id("direct:validate-tenant") + .log("Validating tenant") + .process(exchange -> { + String tenantName = exchange.getIn().getHeader("Platform-TenantId", String.class); + // validation is disabled for now + /*if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { + throw new Exception("Invalid tenant value."); + }*/ + exchange.setProperty(TENANT_NAME, tenantName); + }) + .setHeader("Content-Type", constant("application/json;charset=UTF-8")) + .log("Completed route direct:validate-tenant"); + + + // this route is responsible for editing the incoming records based on configuration + // this step is done to make sure the file format of CSV is not altered and only the data is updated based on config + from("direct:update-incoming-data") + .id("direct:update-incoming-data") + .log("direct:update-incoming-data") + // [LOCAL_FILE_PATH] is already set in [direct:validateFileSyncResponse] route + .setProperty(LOCAL_FILE_PATH, exchangeProperty(FILE_NAME)) + .to("direct:get-transaction-array") + // make sure new data is set under the exchange variable [RESULT_TRANSACTION_LIST] + .process(exchange -> { + String registeringInstituteId = exchange.getProperty(REGISTERING_INSTITUTE_ID, String.class); + String programId = exchange.getProperty(PROGRAM_ID, String.class); + logger.info("Inst id: {}, prog id: {}", registeringInstituteId, programId); + if (!(StringUtils.hasText(registeringInstituteId) && StringUtils.hasText(programId))) { + // this will make sure the file is not updated since there is no update in data + logger.info("Reh or pro is null"); + exchange.setProperty(IS_UPDATED, false); + return; + } + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + logger.info("Size: {}", transactionList.size()); + RegisteringInstitutionConfig registeringInstitutionConfig = budgetAccountConfig + .getByRegisteringInstituteId(registeringInstituteId); + if (registeringInstitutionConfig == null) { + logger.info("Element in nested in config: {}", budgetAccountConfig.getRegisteringInstitutions().get(0).getPrograms().size()); + logger.info("Registering institute id is null"); + exchange.setProperty(IS_UPDATED, false); + return; + } + Program program = registeringInstitutionConfig.getByProgramId(programId); + if (program == null) { + // this will make sure the file is not updated since there is no update in data + logger.info("Program is null"); + exchange.setProperty(IS_UPDATED, false); + return; + } + List resultTransactionList = new ArrayList<>(); + + transactionList.forEach(transaction -> { + transaction.setPayerIdentifierType(program.getIdentifierType()); + transaction.setPayerIdentifier(program.getIdentifierValue()); + resultTransactionList.add(transaction); + try { + logger.info("Txn: {}", objectMapper.writeValueAsString(transaction)); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }); + exchange.setProperty(RESULT_TRANSACTION_LIST, resultTransactionList); + exchange.setProperty(IS_UPDATED, true); + exchange.setProperty(PROGRAM_NAME, program.getName()); + exchange.setProperty(PAYER_IDENTIFIER_TYPE, program.getIdentifierType()); + exchange.setProperty(PAYER_IDENTIFIER_VALUE, program.getIdentifierValue()); + }) + .choice() + // update only when previous(edit function) makes any changes to data + .when(exchange -> exchange.getProperty(IS_UPDATED, Boolean.class)) + // warning: changing this flag can break things + .setProperty(OVERRIDE_HEADER, constant(true)) // default header in CSV file will be used + .to("direct:update-file-v2") + .otherwise() + .log(LoggingLevel.DEBUG, "No update"); + + from("direct:start-batch-process-csv") + .id("direct:start-batch-process-csv") + .log("Starting route direct:start-batch-process-csv") + .to("direct:update-incoming-data") .process(exchange -> { String fileName = exchange.getProperty(FILE_NAME, String.class); String requestId = exchange.getProperty(REQUEST_ID, String.class); String purpose = exchange.getProperty(PURPOSE, String.class); - String batchId = exchange.getProperty(BATCH_ID, String.class); + String batchId = exchange.getProperty(BATCH_ID,String.class); String note = null; if (purpose == null || purpose.isEmpty()) { @@ -158,28 +244,28 @@ private void setup() { logger.info("\n\n Filename: {}", fileName); logger.info("\n\n BatchId: {} ", batchId); - InputStream csvData = exchange.getIn().getBody(InputStream.class); - File file = setupFile(csvData, fileName); - logger.debug("File length {}", file.length()); + File file = new File(fileName); + file.setWritable(true); + file.setReadable(true); + + logger.info("File absolute path: {}", file.getAbsolutePath()); boolean verifyData = verifyData(file); logger.debug("Data verification result {}", verifyData); if (!verifyData) { note = "Invalid data in file data processing stopped"; - } String nm = fileTransferService.uploadFile(file, bucketName); logger.info("File uploaded {}", nm); - // extracting and setting callback Url + //extracting and setting callback Url String callbackUrl = exchange.getIn().getHeader("X-Callback-URL", String.class); - exchange.setProperty(CALLBACK_URL, callbackUrl); + exchange.setProperty(CALLBACK_URL,callbackUrl); List phases = phaseUtils.getValues(); - logger.debug(phases.toString()); + logger.info(phases.toString()); Map variables = new HashMap<>(); - variables.put(FILE_VALIDITY, verifyData); variables.put(BATCH_ID, batchId); variables.put(FILE_NAME, fileName); variables.put(REQUEST_ID, requestId); @@ -189,12 +275,17 @@ private void setup() { variables.put(PHASES, phases); variables.put(PHASE_COUNT, phases.size()); variables.put(NOTE, note); + variables.put(PROGRAM_NAME, exchange.getProperty(PROGRAM_NAME)); + variables.put(PAYER_IDENTIFIER_TYPE, exchange.getProperty(PAYER_IDENTIFIER_TYPE)); + variables.put(PAYER_IDENTIFIER_VALUE, exchange.getProperty(PAYER_IDENTIFIER_VALUE)); + variables.put(REGISTERING_INSTITUTE_ID, exchange.getProperty(REGISTERING_INSTITUTE_ID)); setConfigProperties(variables); JSONObject response = new JSONObject(); try { - String tenantSpecificWorkflowId = workflowId.replace("{dfspid}", exchange.getProperty(TENANT_NAME).toString()); + String tenantSpecificWorkflowId = workflowId.replace("{dfspid}", + exchange.getProperty(TENANT_NAME).toString()); String txnId = zeebeProcessStarter.startZeebeWorkflow(tenantSpecificWorkflowId, "", variables); if (txnId == null || txnId.isEmpty()) { response.put("errorCode", 500); @@ -213,18 +304,25 @@ private void setup() { exchange.getIn().setBody(response.toString()); - }).log("Completed route direct:start-batch-process-csv"); + }) + .log("Completed route direct:start-batch-process-csv"); - from("direct:start-batch-process-raw").id("direct:start-batch-process-raw").log("Starting route direct:start-batch-process-raw") + from("direct:start-batch-process-raw") + .id("direct:start-batch-process-raw") + .log("Starting route direct:start-batch-process-raw") .process(exchange -> { JSONObject response = new JSONObject(); response.put("batch_id", UUID.randomUUID().toString()); response.put("request_id", UUID.randomUUID().toString()); response.put("status", "queued"); exchange.getIn().setBody(response.toString()); - }).log("Completed route direct:start-batch-process-raw"); + }) + .log("Completed route direct:start-batch-process-raw"); - from("direct:executeBatch").id("direct:executeBatch").log("Starting route direct:executeBatch").to("direct:validate-tenant") + from("direct:executeBatch") + .id("direct:executeBatch") + .log("Starting route direct:executeBatch") + .to("direct:validate-tenant") .process(exchange -> { String filename = exchange.getIn().getHeader("filename", String.class); String requestId = exchange.getIn().getHeader("X-CorrelationID", String.class); @@ -234,10 +332,13 @@ private void setup() { exchange.setProperty(REQUEST_ID, requestId); exchange.setProperty(PURPOSE, purpose); exchange.setProperty(BATCH_REQUEST_TYPE, type); - }).choice().when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("raw")) + }) + .choice() + .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("raw")) .to("direct:start-batch-process-raw") - .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("csv")).unmarshal() - .mimeMultipart("multipart/*").to("direct:start-batch-process-csv").otherwise() + .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("csv")) + .to("direct:start-batch-process-csv") + .otherwise() .setBody(exchange -> getUnsupportedTypeJson(exchange.getProperty(BATCH_REQUEST_TYPE, String.class)).toString()) .log("Completed execution of route rest:POST:/batchtransactions"); @@ -249,11 +350,14 @@ private void setup() { }); + from("direct:validateFileSyncResponse").id("direct:validateFileSyncResponse").log("Starting route direct:validateFileSyncResponse") .process(exchange -> { - String fileName = System.currentTimeMillis() + "_" + exchange.getIn().getHeader("fileName", String.class); - InputStream csvData = exchange.getIn().getBody(InputStream.class); - File file = setupFile(csvData, fileName); + // move this logic to spring + String fileName = exchange.getIn().getHeader(FILE_NAME, String.class); + File file = new File(fileName); + + // check the file structure int fileSize = (int) file.length(); if (fileSize > csvSize) { setErrorResponse(exchange, 400, "File too big", @@ -262,23 +366,19 @@ private void setup() { setErrorResponse(exchange, 400, "Invalid file structure", "The file uploaded contains wrong structure." + " Please upload correct file columns and try again."); } else { - logger.debug("Filename: {}", fileName); + logger.info("Filename: {}", fileName); setResponse(exchange, 200); } }).log("Completed route direct:validateFileSyncResponse"); from("direct:validate-file").id("direct:validate-file").log("Starting route direct:validate-file").process(exchange -> { - System.setProperty("mail.mime.multipart.ignoreexistingboundaryparameter", "true"); - System.setProperty("mail.mime.multipart.allowempty", "true"); - InputStream inputStreams = exchange.getIn().getBody(InputStream.class); - MimeBodyPart mimeMessage = new MimeBodyPart(inputStreams); - DataHandler dh = mimeMessage.getDataHandler(); - logger.debug("File name: {} ", dh.getName()); + File f = new File(exchange.getIn().getHeader(FILE_NAME, String.class)); + logger.info("File name: {} ", f.getName()); Tika tika = new Tika(); - String fileType = tika.detect(dh.getName()); - logger.debug("File type: {} ", fileType); - if (dh.getName() == null || dh.getName().isEmpty()) { + String fileType = tika.detect(f.getName()); + logger.info("File type: {} ", fileType); + if (f.getName().isEmpty()) { setErrorResponse(exchange, 400, "File not uploaded", "There was no fie uploaded with the request. " + "Please upload a file and try again."); } else if (!fileType.equalsIgnoreCase("text/csv")) { @@ -289,6 +389,7 @@ private void setup() { } }); + } private boolean verifyData(File file) throws IOException { @@ -300,7 +401,7 @@ private boolean verifyData(File file) throws IOException { while ((line = br.readLine()) != null) { String[] row = line.split(","); if (row.length != columnNames.size()) { - logger.debug("Row invalid {} {}", row.length, columnNames.size()); + logger.info("Row invalid {} {}", row.length, columnNames.size()); return false; } if (!verifyRow(row)) { @@ -317,14 +418,14 @@ private boolean verifyRow(String[] row) { int j = row[i].indexOf("MSISDN"); if (!(j == row.length)) { if (!row[j + 1].matches("^[0-9]*$")) { - logger.debug("MSISDN invalid"); + logger.info("MSISDN invalid"); return false; } } } else if (row[i].contains("amount")) { int j = row[i].indexOf("amount"); if (!row[j].matches("^[0-9]*$")) { - logger.debug("Amount invalid"); + logger.info("Amount invalid"); return false; } @@ -333,18 +434,6 @@ private boolean verifyRow(String[] row) { return true; } - private File setupFile(InputStream csvStream, String fileName) throws IOException { - String csvData = new BufferedReader(new InputStreamReader(csvStream, StandardCharsets.UTF_8)).lines() - .collect(Collectors.joining("\n")); - File file = new File(fileName); - file.setWritable(true); - file.setReadable(true); - FileWriter fileWriter = new FileWriter(file); - fileWriter.write(csvData); - fileWriter.close(); - return file; - } - private boolean verifyCsv(File csvData) throws IOException { BufferedReader br = new BufferedReader(new FileReader(csvData)); String header = br.readLine(); @@ -356,7 +445,7 @@ private boolean verifyCsv(File csvData) throws IOException { int i = 0; while (i < columns.length) { if (columnNames.contains(columns[i])) { - logger.debug("Column name {} is at index {} ", columns[i], columnNames.indexOf(columns[i])); + logger.info("Column name {} is at index {} ", columns[i], columnNames.indexOf(columns[i])); i++; } else { @@ -379,7 +468,6 @@ private void setErrorResponse(Exchange exchange, int responseCode, String errorI private void setResponse(Exchange exchange, int responseCode) { exchange.getIn().setHeader(Exchange.HTTP_RESPONSE_CODE, responseCode); - } private Map setConfigProperties(Map variables) { @@ -393,9 +481,9 @@ private Map setConfigProperties(Map variables) { variables.put(MAX_STATUS_RETRY, maxThresholdCheckRetry); variables.put(COMPLETION_THRESHOLD, completionThreshold); variables.put(THRESHOLD_DELAY, Utils.getZeebeTimerValue(thresholdCheckDelay)); - variables.put(BULK_NOTIF_SUCCESS, false); - variables.put(BULK_NOTIF_FAILURE, false); - variables.put(MAX_CALLBACK_RETRY, maxCallbackRetry); + variables.put(BULK_NOTIF_SUCCESS,false); + variables.put(BULK_NOTIF_FAILURE,false); + variables.put(MAX_CALLBACK_RETRY,maxCallbackRetry); return variables; } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java index 6376f4c3..5caff8ac 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java @@ -39,8 +39,12 @@ public void configure() throws Exception { exchange.getProperty(COMPLETION_RATE).toString()); callbackUrl = exchange.getProperty(CALLBACK_URL).toString(); return body; - }).toD("${header.callbackUrl}?bridgeEndpoint=true&throwExceptionOnFailure=false").choice() - .when(header("CamelHttpResponseCode").startsWith("2")).log(LoggingLevel.INFO, "Callback sending was successful") + }) + .to("direct:set-jws-signature") + .toD("${header.callbackUrl}?bridgeEndpoint=true&throwExceptionOnFailure=false") + .choice() + .when(header("CamelHttpResponseCode").startsWith("2")) + .log(LoggingLevel.INFO, "Callback sending was successful") .process(exchange -> { List phases = (List) exchange.getProperty(PHASES); @@ -71,8 +75,6 @@ public void configure() throws Exception { exchange.setProperty(ERROR_CODE, exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE)); }); - // for temporary callback simulation - from("rest:post:/simulate").log("Reached Simulation"); } public void eliminatePhases(Exchange exchange, List phases, int phaseCount, int completionRate) { diff --git a/src/main/java/org/mifos/processor/bulk/config/BudgetAccountConfig.java b/src/main/java/org/mifos/processor/bulk/config/BudgetAccountConfig.java new file mode 100644 index 00000000..0bb7930e --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/config/BudgetAccountConfig.java @@ -0,0 +1,24 @@ +package org.mifos.processor.bulk.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; +import java.util.ArrayList; +import java.util.List; + +@Getter +@Setter +@Component +@ConfigurationProperties(prefix = "budget-account") +public class BudgetAccountConfig { + + private List registeringInstitutions = new ArrayList<>(); + + public RegisteringInstitutionConfig getByRegisteringInstituteId(String id) { + return getRegisteringInstitutions().stream() + .filter(p -> p.getId().equals(id)) + .findFirst() + .orElse(null); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/config/Program.java b/src/main/java/org/mifos/processor/bulk/config/Program.java new file mode 100644 index 00000000..f9a4a674 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/config/Program.java @@ -0,0 +1,16 @@ +package org.mifos.processor.bulk.config; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class Program { + + private String id, name, identifierType, identifierValue; + +} diff --git a/src/main/java/org/mifos/processor/bulk/config/RegisteringInstitutionConfig.java b/src/main/java/org/mifos/processor/bulk/config/RegisteringInstitutionConfig.java new file mode 100644 index 00000000..fadb404b --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/config/RegisteringInstitutionConfig.java @@ -0,0 +1,26 @@ +package org.mifos.processor.bulk.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; +import java.util.ArrayList; +import java.util.List; + +@Getter +@Setter +@Component +@ConfigurationProperties(prefix = "budget-account.registering-institutions") +public class RegisteringInstitutionConfig { + + private String id; + private List programs = new ArrayList<>(); + + public Program getByProgramId(String id) { + return getPrograms().stream() + .filter(p -> p.getId().equals(id)) + .findFirst() + .orElse(null); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/file/FileStorageService.java b/src/main/java/org/mifos/processor/bulk/file/FileStorageService.java new file mode 100644 index 00000000..198c0a4e --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/file/FileStorageService.java @@ -0,0 +1,11 @@ +package org.mifos.processor.bulk.file; + +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +@Service +public interface FileStorageService { + + public String save(MultipartFile file); + +} diff --git a/src/main/java/org/mifos/processor/bulk/file/FileStorageServiceImpl.java b/src/main/java/org/mifos/processor/bulk/file/FileStorageServiceImpl.java new file mode 100644 index 00000000..a83e0f61 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/file/FileStorageServiceImpl.java @@ -0,0 +1,26 @@ +package org.mifos.processor.bulk.file; + +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; +import java.util.UUID; + +@Service +public class FileStorageServiceImpl implements FileStorageService { + + private final Path root = Paths.get(""); + + @Override + public String save(MultipartFile file) { + String filename = UUID.randomUUID() + "_" + Objects.requireNonNull(file.getOriginalFilename()); + try { + Files.copy(file.getInputStream(), this.root.resolve(filename)); + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + return filename; + } +} diff --git a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java index e49eb3e5..daaac517 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java +++ b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java @@ -1,9 +1,10 @@ package org.mifos.processor.bulk.schema; -import com.fasterxml.jackson.annotation.JsonIgnore; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.Setter; diff --git a/src/main/java/org/mifos/processor/bulk/utility/Headers.java b/src/main/java/org/mifos/processor/bulk/utility/Headers.java new file mode 100644 index 00000000..69aa9f0f --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/utility/Headers.java @@ -0,0 +1,44 @@ +package org.mifos.processor.bulk.utility; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class Headers { + private Map headers; + + private Headers() { + } + + private void setHeaders(Map headers) { + this.headers = headers; + } + + public Map getHeaders() { + return headers; + } + + public Set getHeadersKey() { + return this.headers.keySet(); + } + + public Object get(String key) { + return this.headers.get(key); + } + + public static class HeaderBuilder { + private Map headers = new HashMap<>(); + + public HeaderBuilder addHeader(String key, Object value) { + headers.put(key, value); + return this; + } + + public Headers build() { + Headers headersClassInstance = new Headers(); + headersClassInstance.setHeaders(this.headers); + + return headersClassInstance; + } + } +} diff --git a/src/main/java/org/mifos/processor/bulk/utility/PhaseUtils.java b/src/main/java/org/mifos/processor/bulk/utility/PhaseUtils.java index b281f8a9..ccbe3539 100644 --- a/src/main/java/org/mifos/processor/bulk/utility/PhaseUtils.java +++ b/src/main/java/org/mifos/processor/bulk/utility/PhaseUtils.java @@ -5,7 +5,7 @@ import org.springframework.stereotype.Component; @Component -@ConfigurationProperties(prefix = "callback-phases") +@ConfigurationProperties(prefix = "callbackphases") public class PhaseUtils { private List values; diff --git a/src/main/java/org/mifos/processor/bulk/utility/SpringWrapperUtil.java b/src/main/java/org/mifos/processor/bulk/utility/SpringWrapperUtil.java new file mode 100644 index 00000000..8b77e818 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/utility/SpringWrapperUtil.java @@ -0,0 +1,20 @@ +package org.mifos.processor.bulk.utility; + +import org.apache.camel.CamelContext; +import org.apache.camel.Exchange; +import org.apache.camel.support.DefaultExchange; +import org.springframework.web.multipart.MultipartFile; + +public class SpringWrapperUtil { + public static Exchange getDefaultWrappedExchange(CamelContext camelContext, + Headers headers) { + Exchange exchange = new DefaultExchange(camelContext); + + // Setting headers + for (String headerKey : headers.getHeadersKey()) { + exchange.getIn().setHeader(headerKey, headers.get(headerKey)); + } + + return exchange; + } +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 8272ab51..91d189b6 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -126,5 +126,7 @@ private ZeebeVariables() {} public static final String NOTE = "note"; public static final String PARTY_LOOKUP_FSP_ID = "partyLookupFspId"; - + public static final String PROGRAM_NAME = "programName"; + public static final String PAYER_IDENTIFIER_TYPE = "payerIdentifierType"; + public static final String PAYER_IDENTIFIER_VALUE = "payerIdentifier"; } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index ef56a688..3d89873c 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -85,7 +85,7 @@ config: enable: true ordering: enable: true - field: "payee_identifier" + field: "payerIdentifier" splitting: enable: false sub-batch-size: 5 @@ -119,6 +119,22 @@ callback-phases: - 80 - 100 +server: + ssl: + key-alias: "tomcat-https" + key-store: "classpath:keystore.jks" + key-store-type: JKS + key-password: "password" + key-store-password: "password" + port: 8443 + + +security: + jws: + enable: true + response: + enable: true + identity-account-mapper: hostname : "https://identity.sandbox.mifos.io" @@ -128,3 +144,13 @@ bulk-processor: csv: columnNames: "id,request_id,payment_mode,payer_identifier_type,payer_identifier,payee_identifier_type,payee_identifier,amount,currency,note" size : 100000 # in bytes + + +budget-account: + registeringInstitutions: + - id: "123" + programs: + - id: "SocialWelfare" + name: "Social Welfare" + identifierType: "ACCOUNT" + identifierValue: "123456789" diff --git a/src/main/resources/keystore.jks b/src/main/resources/keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..f0040c54511f5db99680b8393ef6111a2b4ca7f3 GIT binary patch literal 2272 zcmc(g`8U*!7sux_iw0vIWDI4^R*Yp#b|Y(M2BVREjYefDLL{1mWSeA7*<~q1h7j53 zpvUtdMG;BXtWDXz;n8!x-*cY-;QPb<;q^ZEo^#Lp-gEE%%Ki!n1Oguh@J}E@0{y)` zLrzgcLV|)1#rV%OQA-dA24HB=Jq#}gmof(c1(d)70009)Y0$-~Pl`Y6;IRu5!r9D) z;nu5KQL`@3Z+b~XixlUE2UqUc}tM_MKt9Mx@9W(~hX!m?nKG_m!SR0d$t%%_p-b zezecL)0s2OUs-dq19cG#W18n)Y?jnTNiF|hnFUKD9*x%`lxl$-FX5%6V z+g1^#H-#Mp`?&wbNzVYj;nn(BvDE`DO;yAqn&qQWL*|j@XE)DU-i8Q;>hOjaF*sWx zdb?OA;h;2}UX`VzIQgL}T=FN)UfD24qE=pvi#aj+<#_xJxD%=@l^)5_?vpS=z2@Jg zJ9nDmALj3tN{PNoVWZYZW|>D(+s?v?ExeibTy<^5$5_}B8Gr74KmMUfKRYpkQ?B>0 zj#NtP^78gVI!bHaqQ^@JZ{vXSiMy5Q=VqkmGwpI7a zjJ|X`U%R*m>SCmgaXwqcN)?UVj`9(luY z$lZIbkX!t=>V0y`>p&jAD)BRsfsKntRyPpw{Yf-mn^_BLUq{ciB3J2_yEAUNpL*XC zx{0xqNYRJ8MybIcd`D#Z_xc@tjLuv%zcgGE;!a8~uH=_RDc5-3JJrqh`RKBIMy8|6 zaB6z`XQVuO>CQ7v(+gYG;*td_XTM@P<*++wwvbEJTE#1k%;KrGpx4n+J^{s!Ghs#Z z*%h-14Eu}r`olF*Szb_Wn9yWsfiQ2WD?BvL@p5hkPVR+kbasp7`HeWI^CvQV`2PPYiBi)&4QoQ+Ab6yw`_Qcx>+POVG>p_>~sy-o8_t{LguWIjCSU z+8_|*0fq)i!O*~=1rRU*26KYO`zbkA#3Tz04+p1GoJHIg4*+n(K$z=l+!#&{2nEW4 z1Y45S1Tp--0gmLf!jnn(b9gKK|Fk1WPGeJ&Ii5fvQ`7`7e82O!kZ?0c5}s^!SSo}O z{4H@KxlK&%O$pXEL{ri^HDL_$w+cZb@b;EglID1lIfdYeId+ssT?2!~Xkc*aIE=Q_ zA!;6?`hW5N)fNqq{r?^tr~%!+r*RZ{aYj$~5FEs@aU%N{>nDslU@7 zsEyOSSZ6U|uwZSBTgFR17@Kv;CM1!nc>R_`7eTe5GBN)Q*%Ir``&ZtP#jQs9w$x3Q zg*(E<#iVvrnLIg|ijo#xMgU{{O|k1K+GL^yQ_SGRNeh{_OZW$_2KTnAo74FGP^^Q5 zJ*3(#15TX-SAB=&Y|wwIeBc>gB1E^jQ(y=H0*a+D$1xIzQ?rImY5_AC(|a z>Sp^4z|5uVzK?hQ+r+~yJ2VjhX;kP4Z8WDv7ky)D%@{6;;nIyqm7^n9z{}DNwH5ZJ zq6WXh?!4c$sjR8X>m^Aoj+1n`J<7=-#g#JKK9nYlUk5?mHrF zQJ1$fOmIS3D_J#H#E2axJLT2Eksy&(x=N<*z*bv0y%bH#8ZHx)d-6p+Hxrzwny<`S zY{mNNip33zx~FSP_x@o#pE2QtXXw>D>_D)G1I*TYeY_094Yh}M%Ub-N9I&6N@8+bO zGCyf(@1&((>7*LFVm@fgH`1*cjOI9+Z5O67!}avq4YRo*HKDzJvz+_19M{ n@or^P5_}!D?Ip(*1Yg&p=qaYQu$3FMnWn&u$dr1#Uv Date: Tue, 22 Aug 2023 15:02:31 +0530 Subject: [PATCH 086/156] GOV-255: Added configuration for programId/registeringInstituteId (#94) * Added configuration for programId/registeringInstituteId * Added logic to support programId and registeringInstituteId header * Updated the config for n cross n mapping b/w registeringInstituteId and program * GOV-255.2 Publishing programName, payerIdentifierType and payerIdentifierValue as zeebe variable * Response code changed to 202 --- .../camel/routes/ProcessorStartRoute.java | 20 ++++++++++++++----- .../processor/bulk/schema/Transaction.java | 1 - src/main/resources/application.yaml | 1 - 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 172f65ad..25efd5bd 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -34,8 +34,19 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYER_IDENTIFIER_TYPE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYER_IDENTIFIER_VALUE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.NOTE; -import java.io.*; -import java.util.*; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.ArrayList; import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; @@ -162,7 +173,6 @@ private void setup() { .setHeader("Content-Type", constant("application/json;charset=UTF-8")) .log("Completed route direct:validate-tenant"); - // this route is responsible for editing the incoming records based on configuration // this step is done to make sure the file format of CSV is not altered and only the data is updated based on config from("direct:update-incoming-data") @@ -257,7 +267,7 @@ private void setup() { String nm = fileTransferService.uploadFile(file, bucketName); - logger.info("File uploaded {}", nm); + logger.debug("File uploaded {}", nm); //extracting and setting callback Url String callbackUrl = exchange.getIn().getHeader("X-Callback-URL", String.class); @@ -347,7 +357,7 @@ private void setup() { json.put("PollingPath", "/batch/Summary/" + exchange.getProperty(BATCH_ID)); json.put("SuggestedCallbackSeconds", pollApiTimer); exchange.getIn().setBody(json.toString()); - + exchange.getIn().setHeader(Exchange.HTTP_RESPONSE_CODE, 202); }); diff --git a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java index daaac517..3c56a519 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java +++ b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java @@ -1,6 +1,5 @@ package org.mifos.processor.bulk.schema; - import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 3d89873c..5e573824 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -145,7 +145,6 @@ csv: columnNames: "id,request_id,payment_mode,payer_identifier_type,payer_identifier,payee_identifier_type,payee_identifier,amount,currency,note" size : 100000 # in bytes - budget-account: registeringInstitutions: - id: "123" From 2e2cfcba6ee82548f2ad7a67f08af74b8acdc6ae Mon Sep 17 00:00:00 2001 From: Anover000 <140409886+Anover000@users.noreply.github.com> Date: Fri, 25 Aug 2023 16:17:53 +0530 Subject: [PATCH 087/156] replace byte[] with inputStream (#101) --- .../org/mifos/processor/bulk/HealthCheck.java | 5 +++-- .../processor/bulk/api/ApiOriginFilter.java | 2 +- .../processor/bulk/camel/routes/FileRoute.java | 5 +++-- .../processor/bulk/camel/routes/HealthRoute.java | 9 ++++++--- .../processor/bulk/file/AwsFileTransferImpl.java | 12 +++--------- .../bulk/file/AzureFileTransferImpl.java | 16 +++++++++------- .../processor/bulk/file/FileTransferService.java | 3 ++- .../mifos/processor/bulk/zeebe/ZeebeWorkers.java | 5 +++-- 8 files changed, 30 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/HealthCheck.java b/src/main/java/org/mifos/processor/bulk/HealthCheck.java index 37d91a02..36285d18 100644 --- a/src/main/java/org/mifos/processor/bulk/HealthCheck.java +++ b/src/main/java/org/mifos/processor/bulk/HealthCheck.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import java.io.InputStream; import java.util.UUID; import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; @@ -61,11 +62,11 @@ public void configure() { exchange.setProperty(BATCH_ID, batchId); // TODO: How to get sender information? Hard coded in Channel connector? - byte[] csvFile = fileTransferService.downloadFile(fileName, bucketName); + InputStream csvFileInputStream = fileTransferService.streamFile(fileName, bucketName); CsvSchema schema = CsvSchema.emptySchema().withHeader(); MappingIterator readValues = csvMapper.readerWithSchemaFor(TransactionOlder.class).with(schema) - .readValues(csvFile); + .readValues(csvFileInputStream); while (readValues.hasNext()) { TransactionOlder current = readValues.next(); diff --git a/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java b/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java index f43be522..45837f63 100644 --- a/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java +++ b/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java @@ -22,7 +22,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha HttpServletRequest req = (HttpServletRequest) request; String tenant = req.getHeader("Platform-TenantId"); logger.debug("Tenant Name is : {}", tenant); - logger.info("Client IP Address: {}",req.getRemoteHost()); + logger.info("Client IP Address: {}", req.getRemoteHost()); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java index c4d48b78..658965c6 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java @@ -5,6 +5,7 @@ import java.io.File; import java.io.FileOutputStream; +import java.io.InputStream; import org.mifos.processor.bulk.file.FileTransferService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -31,10 +32,10 @@ public void configure() throws Exception { from("direct:download-file").id("direct:download-file").log("Started download-file route").process(exchange -> { String filename = exchange.getProperty(SERVER_FILE_NAME, String.class); - byte[] csvFile = fileTransferService.downloadFile(filename, bucketName); + InputStream csvFileInputStream = fileTransferService.streamFile(filename, bucketName); File file = new File(filename); try (FileOutputStream fos = new FileOutputStream(file)) { - fos.write(csvFile); + fos.write(csvFileInputStream.read()); } exchange.setProperty(LOCAL_FILE_PATH, file.getAbsolutePath()); logger.info("File downloaded"); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/HealthRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/HealthRoute.java index c9cf0e78..b7d914ff 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/HealthRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/HealthRoute.java @@ -10,8 +10,11 @@ public class HealthRoute extends BaseRouteBuilder { public void configure() throws Exception { // todo remove once camel APIs are migrated to spring - from("rest:GET:/actuator/health/liveness") - .id("rest:GET:/actuator/health/liveness") - .setBody(exchange -> new JSONObject(){{ put("status", "UP"); }}.toString()); + from("rest:GET:/actuator/health/liveness").id("rest:GET:/actuator/health/liveness").setBody(exchange -> new JSONObject() { + + { + put("status", "UP"); + } + }.toString()); } } diff --git a/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java b/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java index 85b17abc..109b11af 100644 --- a/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java +++ b/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java @@ -4,10 +4,10 @@ import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; -import com.amazonaws.util.IOUtils; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -41,16 +41,10 @@ public String uploadFile(File file, String bucketName) { } @Override - public byte[] downloadFile(String fileName, String bucketName) { + public InputStream streamFile(String fileName, String bucketName) { S3Object s3Object = s3Client.getObject(bucketName, fileName); S3ObjectInputStream inputStream = s3Object.getObjectContent(); - try { - byte[] content = IOUtils.toByteArray(inputStream); - return content; - } catch (IOException e) { - e.printStackTrace(); - } - return null; + return inputStream; } @Override diff --git a/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java b/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java index 98067c81..e161be16 100644 --- a/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java +++ b/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java @@ -1,11 +1,12 @@ package org.mifos.processor.bulk.file; import com.azure.storage.blob.BlobClientBuilder; -import com.azure.storage.blob.models.BlobProperties; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; -import java.nio.file.Paths; + +import com.azure.storage.blob.specialized.BlobInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -51,13 +52,13 @@ public String uploadFile(File file, String bucketName) { } @Override - public byte[] downloadFile(String fileName, String bucketName) { + public InputStream streamFile(String fileName, String bucketName) { try { File temp = new File("/temp/" + fileName); - BlobProperties properties = client.containerName(bucketName).blobName(fileName).buildClient().downloadToFile(temp.getPath()); - byte[] content = Files.readAllBytes(Paths.get(temp.getPath())); - temp.delete(); - return content; + BlobInputStream csvInputStream = client.containerName(bucketName).blobName(fileName).buildClient().openInputStream(); +// byte[] content = Files.Paths.get(temp.getPath())); +// temp.delete(); + return csvInputStream; } catch (Exception e) { e.printStackTrace(); } @@ -68,4 +69,5 @@ public byte[] downloadFile(String fileName, String bucketName) { public void deleteFile(String fileName, String bucketName) { client.containerName(bucketName).blobName(fileName).buildClient().delete(); } + } diff --git a/src/main/java/org/mifos/processor/bulk/file/FileTransferService.java b/src/main/java/org/mifos/processor/bulk/file/FileTransferService.java index b4c3a6df..7137367b 100644 --- a/src/main/java/org/mifos/processor/bulk/file/FileTransferService.java +++ b/src/main/java/org/mifos/processor/bulk/file/FileTransferService.java @@ -1,6 +1,7 @@ package org.mifos.processor.bulk.file; import java.io.File; +import java.io.InputStream; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; @@ -11,7 +12,7 @@ public interface FileTransferService { String uploadFile(File file, String bucketName); - byte[] downloadFile(String fileName, String bucketName); + InputStream streamFile(String fileName, String bucketName); void deleteFile(String fileName, String bucketName); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java index a8764ab4..cec0c5d4 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.dataformat.csv.CsvSchema; import io.camunda.zeebe.client.ZeebeClient; +import java.io.InputStream; import java.util.Map; import org.apache.camel.CamelContext; import org.apache.camel.Exchange; @@ -70,10 +71,10 @@ private void workerBulkProcessor() { String fileName = (String) variables.get("fileName"); // TODO: How to get sender information? Hard coded in Channel connector? - byte[] csvFile = fileTransferService.downloadFile(fileName, bucketName); + InputStream csvFileInputStream = fileTransferService.streamFile(fileName, bucketName); CsvSchema schema = CsvSchema.emptySchema().withHeader(); - MappingIterator readValues = csvMapper.readerWithSchemaFor(Transaction.class).with(schema).readValues(csvFile); + MappingIterator readValues = csvMapper.readerWithSchemaFor(Transaction.class).with(schema).readValues(csvFileInputStream); /* * while (readValues.hasNext()) { Transaction current = readValues.next(); current.setBatchId(batchId); if From f70e8267aa478e28997aa2b20c7ebc37a35998e2 Mon Sep 17 00:00:00 2001 From: Ankita Ranjan <92036361+ankita10r@users.noreply.github.com> Date: Wed, 30 Aug 2023 19:19:57 +0530 Subject: [PATCH 088/156] request if to cloent correwlation id (#104) --- .../processor/bulk/camel/routes/InitSubBatchRoute.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index 45f95efc..c8086bad 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -128,8 +128,12 @@ public void configure() throws Exception { int index = exchange.getProperty(Exchange.LOOP_INDEX, Integer.class); List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); Transaction transaction = transactionList.get(index); + exchange.setProperty(REQUEST_ID, transaction.getRequestId()); + logger.info("REQUEST_ID: {}", transaction.getRequestId()); exchange.setProperty(TRANSACTION_LIST_ELEMENT, transaction); - }).setHeader("Platform-TenantId", exchangeProperty(TENANT_NAME)).to("direct:dynamic-payload-setter") + }).setHeader("Platform-TenantId", exchangeProperty(TENANT_NAME)) + .setHeader("X-CorrelationID", exchangeProperty(REQUEST_ID)) + .to("direct:dynamic-payload-setter") .to("direct:external-api-call").to("direct:external-api-response-handler").end() // end loop block .endChoice(); From d5d8f8796d9f6f8ee797d5c7edeb2a9f9b822201 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Sat, 2 Sep 2023 15:32:34 +0530 Subject: [PATCH 089/156] GOV 287 Enabled liveness and readiness for the key auth story (#102) * Enabled liveness and readiness * Removed the actuator camel api * Set the isvalid file tot true * Added log to publish zeebe variable * Bug fixed for passing budget account related header to camel * Clientcorrelation id bug fixed * Publishing client correlation id zeebe variable * Info log changed to debug * Zeebe publish variable log changed to info * Updated the connector common --- build.gradle | 3 +- .../API/definition/BatchTransactions.java | 5 +- .../BatchTransactionsController.java | 17 ++-- .../BulkTransferController.java | 3 +- .../processor/bulk/api/ApiOriginFilter.java | 5 +- .../bulk/camel/config/CamelProperties.java | 3 + .../bulk/camel/routes/BatchStatusRoute.java | 5 +- .../bulk/camel/routes/HealthRoute.java | 20 ----- .../camel/routes/OperationsAuthRoute.java | 3 +- .../camel/routes/ProcessorStartRoute.java | 87 +++++++------------ .../bulk/zeebe/ZeebeProcessStarter.java | 1 + .../processor/bulk/zeebe/ZeebeVariables.java | 2 +- src/main/resources/application.yaml | 16 +++- 13 files changed, 78 insertions(+), 92 deletions(-) delete mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/HealthRoute.java diff --git a/build.gradle b/build.gradle index f91d2dc4..4f429709 100644 --- a/build.gradle +++ b/build.gradle @@ -84,6 +84,7 @@ dependencies { implementation "org.springframework.boot:spring-boot-starter-web:$springBootVersion" implementation "org.springframework:spring-web:5.3.14" implementation 'org.springframework.kafka:spring-kafka:2.8.1' + implementation "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion" // spring test dependency testImplementation "org.springframework.boot:spring-boot-starter-test:$springBootVersion" @@ -107,7 +108,7 @@ dependencies { // miscellaneous dependency implementation 'com.google.code.gson:gson:2.8.9' implementation 'org.json:json:20210307' - implementation 'org.mifos:ph-ee-connector-common:1.5.0-SNAPSHOT' + implementation 'org.mifos:ph-ee-connector-common:1.6.0-rc.1' implementation 'org.apache.camel.springboot:camel-spring-boot-starter:3.4.0' implementation 'org.apache.camel:camel-undertow:3.4.0' implementation 'org.springframework.boot:spring-boot-starter:2.5.2' diff --git a/src/main/java/org/mifos/processor/bulk/API/definition/BatchTransactions.java b/src/main/java/org/mifos/processor/bulk/API/definition/BatchTransactions.java index 90201f7f..371bfcf4 100644 --- a/src/main/java/org/mifos/processor/bulk/API/definition/BatchTransactions.java +++ b/src/main/java/org/mifos/processor/bulk/API/definition/BatchTransactions.java @@ -12,6 +12,7 @@ import java.io.IOException; +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; public interface BatchTransactions { @@ -22,5 +23,7 @@ String batchTransactions(HttpServletResponse httpServletResponse, @RequestHeader @RequestHeader(value = FILE_NAME) String fileName, @RequestHeader(value = PURPOSE) String purpose, @RequestHeader(value = "Type") String type, - @RequestHeader(value = "Platform-TenantId") String tenant) throws IOException; + @RequestHeader(value = HEADER_PLATFORM_TENANT_ID) String tenant, + @RequestHeader(value = HEADER_REGISTERING_INSTITUTE_ID, required = false) String registeringInstitutionId, + @RequestHeader(value = HEADER_PROGRAM_ID, required = false) String programId) throws IOException; } diff --git a/src/main/java/org/mifos/processor/bulk/API/implementation/BatchTransactionsController.java b/src/main/java/org/mifos/processor/bulk/API/implementation/BatchTransactionsController.java index 338152d2..a4b89622 100644 --- a/src/main/java/org/mifos/processor/bulk/API/implementation/BatchTransactionsController.java +++ b/src/main/java/org/mifos/processor/bulk/API/implementation/BatchTransactionsController.java @@ -12,11 +12,14 @@ import org.mifos.processor.bulk.utility.SpringWrapperUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartException; import org.springframework.web.multipart.MultipartFile; import javax.servlet.http.HttpServletResponse; import java.io.IOException; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; @@ -37,19 +40,23 @@ public class BatchTransactionsController implements BatchTransactions { @Override public String batchTransactions(HttpServletResponse httpServletResponse, String requestId, MultipartFile file, String fileName, - String purpose, String type, String tenant) throws IOException { - log.info("Inside api logic"); + String purpose, String type, String tenant, + String registeringInstitutionId, String programId) throws IOException { + log.debug("Inside api logic"); String localFileName = fileStorageService.save(file); Headers headers = new Headers.HeaderBuilder() - .addHeader("X-CorrelationID", requestId) + .addHeader(HEADER_CLIENT_CORRELATION_ID, requestId) .addHeader(PURPOSE,purpose) .addHeader(FILE_NAME,localFileName) .addHeader("Type",type) - .addHeader("Platform-TenantId",tenant) + .addHeader(HEADER_PLATFORM_TENANT_ID,tenant) + .addHeader(HEADER_REGISTERING_INSTITUTE_ID, registeringInstitutionId) + .addHeader(HEADER_PROGRAM_ID, programId) .build(); - + log.debug("Headers passed: {}", headers); Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(), headers); + log.debug("Header in exchange: {}", exchange.getIn().getHeaders()); exchange = producerTemplate.send("direct:post-batch-transactions", exchange); int statusCode = exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE, Integer.class); httpServletResponse.setStatus(statusCode); diff --git a/src/main/java/org/mifos/processor/bulk/API/implementation/BulkTransferController.java b/src/main/java/org/mifos/processor/bulk/API/implementation/BulkTransferController.java index 084e5f85..1f8553f6 100644 --- a/src/main/java/org/mifos/processor/bulk/API/implementation/BulkTransferController.java +++ b/src/main/java/org/mifos/processor/bulk/API/implementation/BulkTransferController.java @@ -13,6 +13,7 @@ import java.io.IOException; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; @@ -34,7 +35,7 @@ public String bulkTransfer(String requestId, MultipartFile file, String fileName .addHeader(PURPOSE,purpose) .addHeader(FILE_NAME,fileName) .addHeader("Type",type) - .addHeader("Platform-TenantId",tenant) + .addHeader(HEADER_PLATFORM_TENANT_ID,tenant) .build(); Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(), headers); diff --git a/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java b/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java index 45837f63..8db48fa3 100644 --- a/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java +++ b/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java @@ -13,6 +13,8 @@ import org.springframework.stereotype.Component; import org.springframework.web.filter.GenericFilterBean; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; + public class ApiOriginFilter extends GenericFilterBean { private Logger logger = LoggerFactory.getLogger(this.getClass()); @@ -20,7 +22,8 @@ public class ApiOriginFilter extends GenericFilterBean { @Override public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { HttpServletRequest req = (HttpServletRequest) request; - String tenant = req.getHeader("Platform-TenantId"); + String tenant = req.getHeader("" + + HEADER_PLATFORM_TENANT_ID); logger.debug("Tenant Name is : {}", tenant); logger.info("Client IP Address: {}", req.getRemoteHost()); } diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index b13d938c..83690f5b 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -75,5 +75,8 @@ private CamelProperties() {} public static final String REGISTERING_INSTITUTE_ID = "registeringInstituteId"; public static final String PROGRAM_ID = "programId"; public static final String IS_UPDATED = "isUpdated"; + public static final String HEADER_PLATFORM_TENANT_ID = "Platform-TenantId"; + public static final String HEADER_CLIENT_CORRELATION_ID = "X-CorrelationID"; + public static final String CLIENT_CORRELATION_ID = "clientCorrelationId"; } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java index d00fc449..3b335fdb 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java @@ -1,7 +1,6 @@ package org.mifos.processor.bulk.camel.routes; -import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_STATUS_FAILED; -import static org.mifos.processor.bulk.camel.config.CamelProperties.OPS_APP_ACCESS_TOKEN; +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_RATE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_CODE; @@ -40,7 +39,7 @@ public void configure() throws Exception { getBaseExternalApiRequestRouteDefinition("batch-summary", HttpRequestMethod.GET) .setHeader(Exchange.REST_HTTP_QUERY, simple("batchId=${exchangeProperty." + BATCH_ID + "}")) .setHeader("Authorization", simple("Bearer ${exchangeProperty." + OPS_APP_ACCESS_TOKEN + "}")) - .setHeader("Platform-TenantId", simple("${exchangeProperty." + TENANT_ID + "}")).process(exchange -> { + .setHeader(HEADER_PLATFORM_TENANT_ID, simple("${exchangeProperty." + TENANT_ID + "}")).process(exchange -> { logger.info(exchange.getIn().getHeaders().toString()); }).toD(operationsAppConfig.batchSummaryUrl + "?bridgeEndpoint=true") .log(LoggingLevel.INFO, "Batch summary API response: \n\n ${body}"); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/HealthRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/HealthRoute.java deleted file mode 100644 index b7d914ff..00000000 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/HealthRoute.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.mifos.processor.bulk.camel.routes; - -import org.json.JSONObject; -import org.springframework.stereotype.Component; - -@Component -public class HealthRoute extends BaseRouteBuilder { - - @Override - public void configure() throws Exception { - - // todo remove once camel APIs are migrated to spring - from("rest:GET:/actuator/health/liveness").id("rest:GET:/actuator/health/liveness").setBody(exchange -> new JSONObject() { - - { - put("status", "UP"); - } - }.toString()); - } -} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/OperationsAuthRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/OperationsAuthRoute.java index 7da3f864..da073cc9 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/OperationsAuthRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/OperationsAuthRoute.java @@ -1,5 +1,6 @@ package org.mifos.processor.bulk.camel.routes; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.OPS_APP_ACCESS_TOKEN; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; @@ -44,7 +45,7 @@ public void configure() throws Exception { simpleF("username=%s&password=%s&grant_type=%s", operationsAppConfig.username, operationsAppConfig.password, "password")) .setHeader("Authorization", constant("Basic Y2xpZW50Og==")) - .setHeader("Platform-TenantId", simple("${exchangeProperty." + TENANT_ID + "}")) + .setHeader(HEADER_PLATFORM_TENANT_ID, simple("${exchangeProperty." + TENANT_ID + "}")) .toD(operationsAppConfig.authUrl + "?bridgeEndpoint=true").log(LoggingLevel.INFO, "Auth response: \n\n ${body}"); /** diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 25efd5bd..12dab01c 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -1,39 +1,8 @@ package org.mifos.processor.bulk.camel.routes; -import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_REQUEST_TYPE; -import static org.mifos.processor.bulk.camel.config.CamelProperties.REGISTERING_INSTITUTE_ID; -import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; -import static org.mifos.processor.bulk.camel.config.CamelProperties.PROGRAM_ID; -import static org.mifos.processor.bulk.camel.config.CamelProperties.IS_UPDATED; -import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; -import static org.mifos.processor.bulk.camel.config.CamelProperties.RESULT_TRANSACTION_LIST; -import static org.mifos.processor.bulk.camel.config.CamelProperties.OVERRIDE_HEADER; -import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.APPROVAL_ENABLED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BULK_NOTIF_FAILURE; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BULK_NOTIF_SUCCESS; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_URL; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_THRESHOLD; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_THRESHOLD_CHECK_ENABLED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FORMATTING_ENABLED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_CALLBACK_RETRY; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_STATUS_RETRY; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_ENABLED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERING_ENABLED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_ENABLED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASES; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASE_COUNT; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_ENABLED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.THRESHOLD_DELAY; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PROGRAM_NAME; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYER_IDENTIFIER_TYPE; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYER_IDENTIFIER_VALUE; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.NOTE; +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; + import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -163,7 +132,7 @@ private void setup() { .id("direct:validate-tenant") .log("Validating tenant") .process(exchange -> { - String tenantName = exchange.getIn().getHeader("Platform-TenantId", String.class); + String tenantName = exchange.getIn().getHeader(HEADER_PLATFORM_TENANT_ID, String.class); // validation is disabled for now /*if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { throw new Exception("Invalid tenant value."); @@ -185,27 +154,27 @@ private void setup() { .process(exchange -> { String registeringInstituteId = exchange.getProperty(REGISTERING_INSTITUTE_ID, String.class); String programId = exchange.getProperty(PROGRAM_ID, String.class); - logger.info("Inst id: {}, prog id: {}", registeringInstituteId, programId); + logger.debug("Inst id: {}, prog id: {}", registeringInstituteId, programId); if (!(StringUtils.hasText(registeringInstituteId) && StringUtils.hasText(programId))) { // this will make sure the file is not updated since there is no update in data - logger.info("Reh or pro is null"); + logger.debug("InstitutionId or programId is null"); exchange.setProperty(IS_UPDATED, false); return; } List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); - logger.info("Size: {}", transactionList.size()); + logger.debug("Size: {}", transactionList.size()); RegisteringInstitutionConfig registeringInstitutionConfig = budgetAccountConfig .getByRegisteringInstituteId(registeringInstituteId); if (registeringInstitutionConfig == null) { - logger.info("Element in nested in config: {}", budgetAccountConfig.getRegisteringInstitutions().get(0).getPrograms().size()); - logger.info("Registering institute id is null"); + logger.debug("Element in nested in config: {}", budgetAccountConfig.getRegisteringInstitutions().get(0).getPrograms().size()); + logger.debug("Registering institute id is null"); exchange.setProperty(IS_UPDATED, false); return; } Program program = registeringInstitutionConfig.getByProgramId(programId); if (program == null) { // this will make sure the file is not updated since there is no update in data - logger.info("Program is null"); + logger.debug("Program is null"); exchange.setProperty(IS_UPDATED, false); return; } @@ -216,7 +185,7 @@ private void setup() { transaction.setPayerIdentifier(program.getIdentifierValue()); resultTransactionList.add(transaction); try { - logger.info("Txn: {}", objectMapper.writeValueAsString(transaction)); + logger.debug("Txn: {}", objectMapper.writeValueAsString(transaction)); } catch (JsonProcessingException e) { throw new RuntimeException(e); } @@ -251,14 +220,14 @@ private void setup() { purpose = "test payment"; } - logger.info("\n\n Filename: {}", fileName); - logger.info("\n\n BatchId: {} ", batchId); + logger.debug("\n\n Filename: {}", fileName); + logger.debug("\n\n BatchId: {} ", batchId); File file = new File(fileName); file.setWritable(true); file.setReadable(true); - logger.info("File absolute path: {}", file.getAbsolutePath()); + logger.debug("File absolute path: {}", file.getAbsolutePath()); boolean verifyData = verifyData(file); logger.debug("Data verification result {}", verifyData); if (!verifyData) { @@ -274,7 +243,7 @@ private void setup() { exchange.setProperty(CALLBACK_URL,callbackUrl); List phases = phaseUtils.getValues(); - logger.info(phases.toString()); + logger.debug(phases.toString()); Map variables = new HashMap<>(); variables.put(BATCH_ID, batchId); variables.put(FILE_NAME, fileName); @@ -285,12 +254,14 @@ private void setup() { variables.put(PHASES, phases); variables.put(PHASE_COUNT, phases.size()); variables.put(NOTE, note); + variables.put(CLIENT_CORRELATION_ID, exchange.getProperty(CLIENT_CORRELATION_ID)); variables.put(PROGRAM_NAME, exchange.getProperty(PROGRAM_NAME)); variables.put(PAYER_IDENTIFIER_TYPE, exchange.getProperty(PAYER_IDENTIFIER_TYPE)); variables.put(PAYER_IDENTIFIER_VALUE, exchange.getProperty(PAYER_IDENTIFIER_VALUE)); variables.put(REGISTERING_INSTITUTE_ID, exchange.getProperty(REGISTERING_INSTITUTE_ID)); + variables.put(IS_FILE_VALID, true); setConfigProperties(variables); - + logger.debug("Zeebe variables published: {}", variables); JSONObject response = new JSONObject(); try { @@ -338,10 +309,16 @@ private void setup() { String requestId = exchange.getIn().getHeader("X-CorrelationID", String.class); String purpose = exchange.getIn().getHeader("Purpose", String.class); String type = exchange.getIn().getHeader("Type", String.class); + String clientCorrelationId = exchange.getIn().getHeader(HEADER_CLIENT_CORRELATION_ID, String.class); + String registeringInstitutionId = exchange.getIn().getHeader(HEADER_REGISTERING_INSTITUTE_ID, String.class); + String programId = exchange.getIn().getHeader(HEADER_PROGRAM_ID, String.class); exchange.setProperty(FILE_NAME, filename); exchange.setProperty(REQUEST_ID, requestId); exchange.setProperty(PURPOSE, purpose); exchange.setProperty(BATCH_REQUEST_TYPE, type); + exchange.setProperty(CLIENT_CORRELATION_ID, clientCorrelationId); + exchange.setProperty(REGISTERING_INSTITUTE_ID, registeringInstitutionId); + exchange.setProperty(PROGRAM_ID, programId); }) .choice() .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("raw")) @@ -376,7 +353,7 @@ private void setup() { setErrorResponse(exchange, 400, "Invalid file structure", "The file uploaded contains wrong structure." + " Please upload correct file columns and try again."); } else { - logger.info("Filename: {}", fileName); + logger.debug("Filename: {}", fileName); setResponse(exchange, 200); } @@ -384,10 +361,10 @@ private void setup() { from("direct:validate-file").id("direct:validate-file").log("Starting route direct:validate-file").process(exchange -> { File f = new File(exchange.getIn().getHeader(FILE_NAME, String.class)); - logger.info("File name: {} ", f.getName()); + logger.debug("File name: {} ", f.getName()); Tika tika = new Tika(); String fileType = tika.detect(f.getName()); - logger.info("File type: {} ", fileType); + logger.debug("File type: {} ", fileType); if (f.getName().isEmpty()) { setErrorResponse(exchange, 400, "File not uploaded", "There was no fie uploaded with the request. " + "Please upload a file and try again."); @@ -411,7 +388,7 @@ private boolean verifyData(File file) throws IOException { while ((line = br.readLine()) != null) { String[] row = line.split(","); if (row.length != columnNames.size()) { - logger.info("Row invalid {} {}", row.length, columnNames.size()); + logger.debug("Row invalid {} {}", row.length, columnNames.size()); return false; } if (!verifyRow(row)) { @@ -428,14 +405,14 @@ private boolean verifyRow(String[] row) { int j = row[i].indexOf("MSISDN"); if (!(j == row.length)) { if (!row[j + 1].matches("^[0-9]*$")) { - logger.info("MSISDN invalid"); + logger.debug("MSISDN invalid"); return false; } } } else if (row[i].contains("amount")) { int j = row[i].indexOf("amount"); if (!row[j].matches("^[0-9]*$")) { - logger.info("Amount invalid"); + logger.debug("Amount invalid"); return false; } @@ -450,12 +427,12 @@ private boolean verifyCsv(File csvData) throws IOException { String[] columns = new String[0]; if (header != null) { columns = header.split(","); - logger.info("Columns in the csv file are {}", Arrays.toString(columns)); + logger.debug("Columns in the csv file are {}", Arrays.toString(columns)); } int i = 0; while (i < columns.length) { if (columnNames.contains(columns[i])) { - logger.info("Column name {} is at index {} ", columns[i], columnNames.indexOf(columns[i])); + logger.debug("Column name {} is at index {} ", columns[i], columnNames.indexOf(columns[i])); i++; } else { diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeProcessStarter.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeProcessStarter.java index a62e69ea..538b8280 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeProcessStarter.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeProcessStarter.java @@ -71,6 +71,7 @@ public String startZeebeWorkflow(String workflowId, String request, Map Date: Sat, 2 Sep 2023 18:33:17 +0530 Subject: [PATCH 090/156] Fixed the regression in bulk (#110) --- .../bulk/camel/routes/ExternalApiCallRoute.java | 2 +- .../processor/bulk/camel/routes/FileRoute.java | 4 ++-- .../bulk/camel/routes/InitSubBatchRoute.java | 12 +++++++++--- .../bulk/camel/routes/ProcessorStartRoute.java | 6 ++++-- .../processor/bulk/file/AwsFileTransferImpl.java | 14 ++++++++++++++ .../bulk/file/AzureFileTransferImpl.java | 16 ++++++++++++++++ .../processor/bulk/file/FileTransferService.java | 2 ++ src/main/resources/application.yaml | 16 ++++++++++++++++ 8 files changed, 64 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java index 8035b40c..fbb282a8 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java @@ -7,7 +7,7 @@ import org.apache.camel.builder.RouteBuilder; import org.springframework.stereotype.Component; -@Component +//@Component public class ExternalApiCallRoute extends RouteBuilder { @Override diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java index 658965c6..0c332f0a 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java @@ -32,10 +32,10 @@ public void configure() throws Exception { from("direct:download-file").id("direct:download-file").log("Started download-file route").process(exchange -> { String filename = exchange.getProperty(SERVER_FILE_NAME, String.class); - InputStream csvFileInputStream = fileTransferService.streamFile(filename, bucketName); + byte[] csvFile = fileTransferService.downloadFile(filename, bucketName); File file = new File(filename); try (FileOutputStream fos = new FileOutputStream(file)) { - fos.write(csvFileInputStream.read()); + fos.write(csvFile); } exchange.setProperty(LOCAL_FILE_PATH, file.getAbsolutePath()); logger.info("File downloaded"); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index c8086bad..46063c88 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -3,6 +3,7 @@ import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_ID_HEADER; import static org.mifos.processor.bulk.camel.config.CamelProperties.EXTERNAL_ENDPOINT; import static org.mifos.processor.bulk.camel.config.CamelProperties.EXTERNAL_ENDPOINT_FAILED; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_CLIENT_CORRELATION_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.IS_PAYMENT_MODE_VALID; import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; import static org.mifos.processor.bulk.camel.config.CamelProperties.OVERRIDE_HEADER; @@ -132,7 +133,6 @@ public void configure() throws Exception { logger.info("REQUEST_ID: {}", transaction.getRequestId()); exchange.setProperty(TRANSACTION_LIST_ELEMENT, transaction); }).setHeader("Platform-TenantId", exchangeProperty(TENANT_NAME)) - .setHeader("X-CorrelationID", exchangeProperty(REQUEST_ID)) .to("direct:dynamic-payload-setter") .to("direct:external-api-call").to("direct:external-api-response-handler").end() // end loop block .endChoice(); @@ -152,9 +152,10 @@ public void configure() throws Exception { from("direct:external-api-response-handler").id("direct:external-api-response-handler") .log("Starting route direct:external-api-response-handler").choice().when(header("CamelHttpResponseCode").isEqualTo(200)) .process(exchange -> { - logger.info("reached here"); + logger.info("INIT_SUB_BATCH_FAILED is false"); exchange.setProperty(INIT_SUB_BATCH_FAILED, false); }).otherwise().process(exchange -> { + logger.info("INIT_SUB_BATCH_FAILED is false"); exchange.setProperty(INIT_SUB_BATCH_FAILED, true); }).endChoice(); @@ -191,8 +192,13 @@ public void configure() throws Exception { .log(LoggingLevel.DEBUG, "Making API call to endpoint ${exchangeProperty.extEndpoint} and body: ${body}") .setHeader(Exchange.CONTENT_TYPE, constant("application/json")) .setHeader(BATCH_ID_HEADER, simple("${exchangeProperty." + BATCH_ID + "}")) + .setHeader(HEADER_CLIENT_CORRELATION_ID, simple("${exchangeProperty." + REQUEST_ID + "}")) + .process(exchange -> { + log.debug("Variables: {}", exchange.getProperties()); + log.debug("Emergency: {}", exchange.getIn().getHeaders()); + }) .toD(ChannelURL + "${exchangeProperty.extEndpoint}" + "?bridgeEndpoint=true&throwExceptionOnFailure=false") - .log(LoggingLevel.DEBUG, "Response body: ${body}").otherwise().endChoice(); + .log(LoggingLevel.INFO, "Response body: ${body}").otherwise().endChoice(); from("direct:validate-payment-mode").id("direct:validate-payment-mode").log("Starting route direct:validate-payment-mode") .process(exchange -> { diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 12dab01c..a17ea337 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -2,7 +2,6 @@ import static org.mifos.processor.bulk.camel.config.CamelProperties.*; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; - import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -203,7 +202,7 @@ private void setup() { .setProperty(OVERRIDE_HEADER, constant(true)) // default header in CSV file will be used .to("direct:update-file-v2") .otherwise() - .log(LoggingLevel.DEBUG, "No update"); + .log(LoggingLevel.INFO, "No update"); from("direct:start-batch-process-csv") .id("direct:start-batch-process-csv") @@ -261,7 +260,10 @@ private void setup() { variables.put(REGISTERING_INSTITUTE_ID, exchange.getProperty(REGISTERING_INSTITUTE_ID)); variables.put(IS_FILE_VALID, true); setConfigProperties(variables); + logger.debug("Zeebe variables published: {}", variables); + log.debug("Variables published to zeebe: {}", variables); + JSONObject response = new JSONObject(); try { diff --git a/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java b/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java index 109b11af..571c2705 100644 --- a/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java +++ b/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java @@ -8,6 +8,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import com.amazonaws.util.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -24,6 +25,19 @@ public class AwsFileTransferImpl implements FileTransferService { @Autowired private AmazonS3 s3Client; + @Override + public byte[] downloadFile(String fileName, String bucketName) { + S3Object s3Object = s3Client.getObject(bucketName, fileName); + S3ObjectInputStream inputStream = s3Object.getObjectContent(); + try { + byte[] content = IOUtils.toByteArray(inputStream); + return content; + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + @Override public String uploadFile(MultipartFile file, String bucketName) { diff --git a/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java b/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java index e161be16..421ba7cc 100644 --- a/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java +++ b/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java @@ -5,7 +5,9 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; +import java.nio.file.Paths; +import com.azure.storage.blob.models.BlobProperties; import com.azure.storage.blob.specialized.BlobInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,6 +27,20 @@ public class AzureFileTransferImpl implements FileTransferService { @Autowired BlobClientBuilder client; + @Override + public byte[] downloadFile(String fileName, String bucketName) { + try { + File temp = new File("/temp/" + fileName); + BlobProperties properties = client.containerName(bucketName).blobName(fileName).buildClient().downloadToFile(temp.getPath()); + byte[] content = Files.readAllBytes(Paths.get(temp.getPath())); + temp.delete(); + return content; + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + @Override public String uploadFile(MultipartFile file, String bucketName) { diff --git a/src/main/java/org/mifos/processor/bulk/file/FileTransferService.java b/src/main/java/org/mifos/processor/bulk/file/FileTransferService.java index 7137367b..f7e9f3ee 100644 --- a/src/main/java/org/mifos/processor/bulk/file/FileTransferService.java +++ b/src/main/java/org/mifos/processor/bulk/file/FileTransferService.java @@ -8,6 +8,8 @@ @Service public interface FileTransferService { + byte[] downloadFile(String fileName, String bucketName); + String uploadFile(MultipartFile file, String bucketName); String uploadFile(File file, String bucketName); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index c42e12c8..214d5ef5 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -163,3 +163,19 @@ management: enabled: true readiness: enabled: true + +payment-mode: + mappings: + - id: "GSMA" + type: "PAYMENT" + endpoint: "/channel/gsma/transfer" + - id: "MOJALOOP" + type: "PAYMENT" + endpoint: "/channel/transfer" + - id: "SLCB" + type: "BULK" + endpoint: "bulk_connector_{MODE}-{dfspid}" + - id: "CLOSEDLOOP" + type: "BULK" + endpoint: "bulk_connector_{MODE}-{dfspid}" + debulkingDfspid: "lion" From 8987d478507beae2f04d604fca7051d04c6f4687 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Mon, 4 Sep 2023 15:26:42 +0530 Subject: [PATCH 091/156] PHEE-415 Implemented the batch REST Api and Optimised imports and updated the batch API (#105) * Implemented the batch REST Api and Optimised imports and updated the batch API * Header parsing bug fixed in camel * Reverted the codestyle changes * Reverted the codestyle changes * Reverted the codestyle changes * Removed code Formatting * Bug fixed in converting the batchRequest to txn object * Added header validation * Wired raw request with csv request --- .circleci/config.yml | 11 +- README.md | 2 +- build.gradle | 2 +- .../API/definition/BatchTransactions.java | 24 +-- .../bulk/API/definition/BulkTransfer.java | 10 +- .../BatchTransactionsController.java | 150 ++++++++++++++---- .../BulkTransferController.java | 20 +-- .../camel/routes/ExternalApiCallRoute.java | 1 - .../camel/routes/FileProcessingRoute.java | 19 +-- .../camel/routes/ProcessorStartRoute.java | 109 +++++-------- .../bulk/file/AzureFileTransferImpl.java | 4 +- .../bulk/file/FileStorageService.java | 6 + .../bulk/file/FileStorageServiceImpl.java | 29 ++++ .../processor/bulk/format/EntityMapper.java | 14 ++ .../bulk/format/RestRequestConvertor.java | 78 +++++++++ .../bulk/schema/BatchRequestDTO.java | 20 +++ .../bulk/schema/CamelApiResponse.java | 14 ++ .../mifos/processor/bulk/schema/Party.java | 16 ++ .../processor/bulk/schema/PartyType.java | 26 +++ .../processor/bulk/utility/CsvWriter.java | 32 ++++ .../processor/bulk/zeebe/ZeebeVariables.java | 3 + src/main/resources/application.yaml | 2 +- 22 files changed, 443 insertions(+), 149 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/format/EntityMapper.java create mode 100644 src/main/java/org/mifos/processor/bulk/format/RestRequestConvertor.java create mode 100644 src/main/java/org/mifos/processor/bulk/schema/BatchRequestDTO.java create mode 100644 src/main/java/org/mifos/processor/bulk/schema/CamelApiResponse.java create mode 100644 src/main/java/org/mifos/processor/bulk/schema/Party.java create mode 100644 src/main/java/org/mifos/processor/bulk/schema/PartyType.java create mode 100644 src/main/java/org/mifos/processor/bulk/utility/CsvWriter.java diff --git a/.circleci/config.yml b/.circleci/config.yml index ff638dda..3e699b75 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ jobs: executor: docker-executor environment: JVM_OPTS: -Xmx512m - TERM: dumb + TERM: dumb GITHUB_TOKEN: ${GITHUB_TOKEN} # Add the GitHub token as an environment variable steps: @@ -40,7 +40,7 @@ jobs: docker push "openmf/ph-ee-bulk-processor:$IMAGE_TAG" # when: always # The job will be executed even if there's no match for the tag filter - + build_and_push_latest_image: executor: docker-executor environment: @@ -68,7 +68,7 @@ jobs: # Push the Docker image to DockerHub - run: name: Push Docker image to DockerHub - command: docker push openmf/ph-ee-bulk-processor:latest + command: docker push openmf/ph-ee-bulk-processor:latest workflows: version: 2 @@ -79,8 +79,7 @@ workflows: tags: only: /^v\d+\.\d+\.\d+$/ # Match tags in the format v1.2.3 context: - - DOCKER + - DOCKER - build_and_push_latest_image: context: - - DOCKER - + - DOCKER diff --git a/README.md b/README.md index 9e3835a6..3840e736 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ server: key-store-password: "" port: 8443 ``` -#### NOTE: For disabling TLS, change the port to "8080" and add null values for all the "ssl" related fields. +#### NOTE: For disabling TLS, change the port to "8080" and add null values for all the "ssl" related fields. ## Checkstyle Use below command to execute the checkstyle test. diff --git a/build.gradle b/build.gradle index 4f429709..5e39a38e 100644 --- a/build.gradle +++ b/build.gradle @@ -108,7 +108,7 @@ dependencies { // miscellaneous dependency implementation 'com.google.code.gson:gson:2.8.9' implementation 'org.json:json:20210307' - implementation 'org.mifos:ph-ee-connector-common:1.6.0-rc.1' + implementation 'org.mifos:ph-ee-connector-common:1.6.1-rc.1' implementation 'org.apache.camel.springboot:camel-spring-boot-starter:3.4.0' implementation 'org.apache.camel:camel-undertow:3.4.0' implementation 'org.springframework.boot:spring-boot-starter:2.5.2' diff --git a/src/main/java/org/mifos/processor/bulk/API/definition/BatchTransactions.java b/src/main/java/org/mifos/processor/bulk/API/definition/BatchTransactions.java index 371bfcf4..e287bfcc 100644 --- a/src/main/java/org/mifos/processor/bulk/API/definition/BatchTransactions.java +++ b/src/main/java/org/mifos/processor/bulk/API/definition/BatchTransactions.java @@ -8,22 +8,28 @@ import org.springframework.web.multipart.MultipartFile; import javax.mail.Multipart; +import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_CLIENT_CORRELATION_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; public interface BatchTransactions { - @PostMapping(value = "/batchtransactions", produces="application/json") - String batchTransactions(HttpServletResponse httpServletResponse, @RequestHeader(value = "X-CorrelationID") String requestId, - @RequestParam("data") MultipartFile file, - @RequestHeader(value = FILE_NAME) String fileName, - @RequestHeader(value = PURPOSE) String purpose, - @RequestHeader(value = "Type") String type, - @RequestHeader(value = HEADER_PLATFORM_TENANT_ID) String tenant, - @RequestHeader(value = HEADER_REGISTERING_INSTITUTE_ID, required = false) String registeringInstitutionId, - @RequestHeader(value = HEADER_PROGRAM_ID, required = false) String programId) throws IOException; + @PostMapping(value = "/batchtransactions", produces = "application/json") + String batchTransactions( + HttpServletRequest httpServletRequest, + HttpServletResponse httpServletResponse, + @RequestHeader(value = HEADER_CLIENT_CORRELATION_ID) String requestId, + @RequestHeader(value = FILE_NAME, required = false) String fileName, + @RequestHeader(value = PURPOSE) String purpose, + @RequestHeader(value = HEADER_TYPE) String type, + @RequestHeader(value = HEADER_PLATFORM_TENANT_ID) String tenant, + @RequestHeader(value = HEADER_REGISTERING_INSTITUTE_ID, required = false) String registeringInstitutionId, + @RequestHeader(value = HEADER_PROGRAM_ID, required = false) String programId) throws IOException; + } diff --git a/src/main/java/org/mifos/processor/bulk/API/definition/BulkTransfer.java b/src/main/java/org/mifos/processor/bulk/API/definition/BulkTransfer.java index 328c8c28..5e0a3325 100644 --- a/src/main/java/org/mifos/processor/bulk/API/definition/BulkTransfer.java +++ b/src/main/java/org/mifos/processor/bulk/API/definition/BulkTransfer.java @@ -11,11 +11,11 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; public interface BulkTransfer { + + @Deprecated @PostMapping(value = "/bulk/transfer/{requestId}/{fileName}", produces = "application/json") String bulkTransfer(@RequestHeader(value = "X-CorrelationID", required = false) String requestId, - @RequestParam("data") MultipartFile file, - @RequestHeader(value = FILE_NAME, required = false) String fileName, - @RequestHeader(value = PURPOSE, required = false) String purpose, - @RequestHeader(value = "Type", required = false) String type, - @RequestHeader(value = "Platform-TenantId") String tenant) throws IOException; + @RequestParam("data") MultipartFile file, @RequestHeader(value = FILE_NAME, required = false) String fileName, + @RequestHeader(value = PURPOSE, required = false) String purpose, @RequestHeader(value = "Type", required = false) String type, + @RequestHeader(value = "Platform-TenantId") String tenant) throws IOException; } diff --git a/src/main/java/org/mifos/processor/bulk/API/implementation/BatchTransactionsController.java b/src/main/java/org/mifos/processor/bulk/API/implementation/BatchTransactionsController.java index a4b89622..26a68cdd 100644 --- a/src/main/java/org/mifos/processor/bulk/API/implementation/BatchTransactionsController.java +++ b/src/main/java/org/mifos/processor/bulk/API/implementation/BatchTransactionsController.java @@ -1,27 +1,46 @@ package org.mifos.processor.bulk.api.implementation; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.camel.Exchange; import org.apache.camel.ProducerTemplate; +import org.apache.commons.io.IOUtils; import org.json.JSONObject; +import org.mifos.connector.common.interceptor.JWSUtil; import org.mifos.processor.bulk.api.definition.BatchTransactions; import org.mifos.processor.bulk.file.FileStorageService; +import org.mifos.processor.bulk.format.RestRequestConvertor; +import org.mifos.processor.bulk.schema.BatchRequestDTO; +import org.mifos.processor.bulk.schema.CamelApiResponse; +import org.mifos.processor.bulk.schema.Transaction; +import org.mifos.processor.bulk.utility.CsvWriter; import org.mifos.processor.bulk.utility.Headers; import org.mifos.processor.bulk.utility.SpringWrapperUtil; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartException; -import org.springframework.web.multipart.MultipartFile; +import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; - import static org.mifos.processor.bulk.camel.config.CamelProperties.*; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PROGRAM_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_CLIENT_CORRELATION_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_PLATFORM_TENANT_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_TYPE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; @Slf4j @RestController @@ -36,41 +55,118 @@ public class BatchTransactionsController implements BatchTransactions { @Autowired FileStorageService fileStorageService; + @Autowired + RestRequestConvertor restRequestConvertor; + + @Value("#{'${tenants}'.split(',')}") + protected List tenants; + @Autowired + private CsvMapper csvMapper; + @SneakyThrows @Override - public String batchTransactions(HttpServletResponse httpServletResponse, - String requestId, MultipartFile file, String fileName, - String purpose, String type, String tenant, - String registeringInstitutionId, String programId) throws IOException { - log.debug("Inside api logic"); - String localFileName = fileStorageService.save(file); - Headers headers = new Headers.HeaderBuilder() + public String batchTransactions( + HttpServletRequest httpServletRequest, + HttpServletResponse httpServletResponse, + String requestId, + String fileName, + String purpose, + String type, + String tenant, + String registeringInstitutionId, + String programId) { + + log.info("Inside api logic"); + Headers.HeaderBuilder headerBuilder = new Headers.HeaderBuilder() .addHeader(HEADER_CLIENT_CORRELATION_ID, requestId) - .addHeader(PURPOSE,purpose) - .addHeader(FILE_NAME,localFileName) - .addHeader("Type",type) - .addHeader(HEADER_PLATFORM_TENANT_ID,tenant) + .addHeader(PURPOSE, purpose) + .addHeader(HEADER_TYPE, type) + .addHeader(HEADER_PLATFORM_TENANT_ID, tenant) .addHeader(HEADER_REGISTERING_INSTITUTE_ID, registeringInstitutionId) - .addHeader(HEADER_PROGRAM_ID, programId) - .build(); - log.debug("Headers passed: {}", headers); - Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(), - headers); - log.debug("Header in exchange: {}", exchange.getIn().getHeaders()); + .addHeader(HEADER_PROGRAM_ID, programId); + + Optional validationResponse = isValidRequest(httpServletRequest, fileName, type); + if (validationResponse.isPresent()) { + httpServletResponse.setStatus(httpServletResponse.SC_BAD_REQUEST); + return validationResponse.get(); + } + + if (JWSUtil.isMultipartRequest(httpServletRequest)) { + log.info("This is file based request"); + String localFileName = fileStorageService.save(JWSUtil.parseFormData(httpServletRequest), fileName); + Headers headers = headerBuilder.addHeader(FILE_NAME, localFileName).build(); + log.info("Headers passed: {}", headers.getHeaders()); + + CamelApiResponse response = sendRequestToCamel(headers); + httpServletResponse.setStatus(response.getStatus()); + return response.getBody(); + } else { + log.info("This is json based request"); + String jsonString = IOUtils.toString(httpServletRequest.getInputStream(), Charset.defaultCharset()); + List batchRequestDTOList = objectMapper.readValue(jsonString, new TypeReference<>() {}); + List transactionList = restRequestConvertor.convertListFrom(batchRequestDTOList); + + String localFileName = UUID.randomUUID() + ".csv"; + CsvWriter.writeToCsv(transactionList, Transaction.class, csvMapper, true, localFileName); + Headers headers = headerBuilder + .addHeader(HEADER_TYPE, "csv") + .addHeader(FILE_NAME, localFileName).build(); + + CamelApiResponse response = sendRequestToCamel(headers); + httpServletResponse.setStatus(response.getStatus()); + return response.getBody(); + } + } + + @ExceptionHandler({ MultipartException.class }) + public String handleMultipartException(HttpServletResponse httpServletResponse) { + httpServletResponse.setStatus(httpServletResponse.SC_BAD_REQUEST); + return getErrorResponse("File not uploaded", "There was no fie uploaded with the request. " + + "Please upload a file and try again.", 400); + } + + private CamelApiResponse sendRequestToCamel(Headers headers) { + Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(), headers); exchange = producerTemplate.send("direct:post-batch-transactions", exchange); int statusCode = exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE, Integer.class); - httpServletResponse.setStatus(statusCode); - return exchange.getIn().getBody(String.class); + String body = exchange.getIn().getBody(String.class); + return new CamelApiResponse(body, statusCode); } - @ExceptionHandler({MultipartException.class}) - public String handleMultipartException(HttpServletResponse httpServletResponse) { + private String getErrorResponse(String information, String description, int code) { JSONObject json = new JSONObject(); - json.put("Error Information: ", "File not uploaded"); - json.put("Error Description : ", "There was no fie uploaded with the request. " + - "Please upload a file and try again."); - httpServletResponse.setStatus(httpServletResponse.SC_BAD_REQUEST); + json.put("errorInformation", "File not uploaded"); + json.put("errorDescription", "There was no fie uploaded with the request. " + "Please upload a file and try again."); + json.put("errorCode", code); return json.toString(); } + // validates the request header, and return errorJson string if the request is invalid else an empty optional + private Optional isValidRequest(HttpServletRequest httpServletRequest, + String fileName, + String type) { + + Optional response = Optional.empty(); + if ((JWSUtil.isMultipartRequest(httpServletRequest) && !type.equalsIgnoreCase("csv")) || + (!JWSUtil.isMultipartRequest(httpServletRequest) && !type.equalsIgnoreCase("raw"))) { + String errorJson = getErrorResponse("Type mismatch", + "The value of the header \"" + HEADER_TYPE + + "\" doesn't match with the request content-type", 400); + response = Optional.of(errorJson); + + } + if (JWSUtil.isMultipartRequest(httpServletRequest) && fileName.isEmpty()) { + String errorJson = getErrorResponse("Header can't be empty", + "If the request is of type csv, the header \"" + + FILE_NAME + "\"can't be empty", 400); + response = Optional.of(errorJson); + } + if (!type.equalsIgnoreCase("raw") && !type.equalsIgnoreCase("csv")) { + String errorJson = getErrorResponse("Invalid TYPE header value passed", + "The value of the header \"" + HEADER_TYPE + + "\" can be \"[raw,csv]\" but is " + type, 400); + response = Optional.of(errorJson); + } + return response; + } } diff --git a/src/main/java/org/mifos/processor/bulk/API/implementation/BulkTransferController.java b/src/main/java/org/mifos/processor/bulk/API/implementation/BulkTransferController.java index 1f8553f6..4606a73a 100644 --- a/src/main/java/org/mifos/processor/bulk/API/implementation/BulkTransferController.java +++ b/src/main/java/org/mifos/processor/bulk/API/implementation/BulkTransferController.java @@ -10,15 +10,16 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; - import java.io.IOException; - import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_CLIENT_CORRELATION_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_TYPE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; @RestController public class BulkTransferController implements BulkTransfer { + @Autowired private ProducerTemplate producerTemplate; @@ -29,16 +30,11 @@ public class BulkTransferController implements BulkTransfer { FileStorageService fileStorageService; @Override - public String bulkTransfer(String requestId, MultipartFile file, String fileName, String purpose, String type, String tenant) throws IOException { - Headers headers = new Headers.HeaderBuilder() - .addHeader("X-CorrelationID", requestId) - .addHeader(PURPOSE,purpose) - .addHeader(FILE_NAME,fileName) - .addHeader("Type",type) - .addHeader(HEADER_PLATFORM_TENANT_ID,tenant) - .build(); - Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(), - headers); + public String bulkTransfer(String requestId, MultipartFile file, String fileName, String purpose, String type, String tenant) + throws IOException { + Headers headers = new Headers.HeaderBuilder().addHeader(HEADER_CLIENT_CORRELATION_ID, requestId).addHeader(PURPOSE, purpose) + .addHeader(FILE_NAME, fileName).addHeader(HEADER_TYPE, type).addHeader(HEADER_PLATFORM_TENANT_ID, tenant).build(); + Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(), headers); fileStorageService.save(file); producerTemplate.send("direct:post-bulk-transfer", exchange); return exchange.getIn().getBody(String.class); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java index fbb282a8..b2d645d7 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java @@ -5,7 +5,6 @@ import org.apache.camel.LoggingLevel; import org.apache.camel.builder.RouteBuilder; -import org.springframework.stereotype.Component; //@Component public class ExternalApiCallRoute extends RouteBuilder { diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java index 9959bcc4..e0e7e53d 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java @@ -20,6 +20,7 @@ import java.util.List; import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.schema.TransactionResult; +import org.mifos.processor.bulk.utility.CsvWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -77,7 +78,7 @@ public void configure() { // getting header Boolean overrideHeader = exchange.getProperty(OVERRIDE_HEADER, Boolean.class); - csvWriter(transactionList, TransactionResult.class, csvMapper, overrideHeader, filepath); + CsvWriter.writeToCsv(transactionList, TransactionResult.class, csvMapper, overrideHeader, filepath); }).log("Update complete"); /** @@ -113,20 +114,4 @@ public void configure() { } }); } - - private void csvWriter(List data, Class tClass, CsvMapper csvMapper, boolean overrideHeader, String filepath) - throws IOException { - CsvSchema csvSchema = csvMapper.schemaFor(tClass); - if (overrideHeader) { - csvSchema = csvSchema.withHeader(); - } else { - csvSchema = csvSchema.withoutHeader(); - } - - File file = new File(filepath); - SequenceWriter writer = csvMapper.writerWithSchemaFor(tClass).with(csvSchema).writeValues(file); - for (T object : data) { - writer.write(object); - } - } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index a17ea337..dee022ee 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -2,6 +2,8 @@ import static org.mifos.processor.bulk.camel.config.CamelProperties.*; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_CLIENT_CORRELATION_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -92,40 +94,26 @@ public void configure() { private void setup() { - from("direct:post-batch-transactions") - .id("rest:POST:/batchtransactions") - .log("Starting route rest:POST:/batchtransactions") - .to("direct:validate-file") - .choice() - .when(header("CamelHttpResponseCode").isNotEqualTo("200")) - .log(LoggingLevel.ERROR, "File upload failed") - .otherwise() - .process(exchange -> { + from("direct:post-batch-transactions").id("rest:POST:/batchtransactions").log("Starting route rest:POST:/batchtransactions") + .to("direct:validate-file").choice().when(header("CamelHttpResponseCode").isNotEqualTo("200")) + .log(LoggingLevel.ERROR, "File upload failed").otherwise().process(exchange -> { String batchId = UUID.randomUUID().toString(); exchange.setProperty(BATCH_ID, batchId); - }) - .to("direct:validateFileSyncResponse") - .choice() - .when(header("CamelHttpResponseCode").isNotEqualTo("200")) - .log(LoggingLevel.ERROR, "File upload failed").otherwise() - .wireTap("direct:executeBatch").to("direct:pollingOutput").endChoice().endChoice(); - - from("direct:post-bulk-transfer") - .unmarshal().mimeMultipart("multipart/*") - .to("direct:validate-tenant") - .process(exchange -> { - String fileName = System.currentTimeMillis() + "_" + exchange.getIn().getHeader("fileName", String.class); - String requestId = exchange.getIn().getHeader("requestId", String.class); - String purpose = exchange.getIn().getHeader("purpose", String.class); - String batchId = UUID.randomUUID().toString(); - exchange.setProperty(BATCH_ID,batchId); - exchange.setProperty(FILE_NAME, fileName); - exchange.setProperty(REQUEST_ID, requestId); - exchange.setProperty(PURPOSE, purpose); - }) - .wireTap("direct:start-batch-process-csv") - .to("direct:pollingOutput"); + }).to("direct:validateFileSyncResponse").choice().when(header("CamelHttpResponseCode").isNotEqualTo("200")) + .log(LoggingLevel.ERROR, "File upload failed").otherwise().wireTap("direct:executeBatch").to("direct:pollingOutput") + .endChoice().endChoice(); + + from("direct:post-bulk-transfer").unmarshal().mimeMultipart("multipart/*").to("direct:validate-tenant").process(exchange -> { + String fileName = System.currentTimeMillis() + "_" + exchange.getIn().getHeader("fileName", String.class); + String requestId = exchange.getIn().getHeader("requestId", String.class); + String purpose = exchange.getIn().getHeader("purpose", String.class); + String batchId = UUID.randomUUID().toString(); + exchange.setProperty(BATCH_ID, batchId); + exchange.setProperty(FILE_NAME, fileName); + exchange.setProperty(REQUEST_ID, requestId); + exchange.setProperty(PURPOSE, purpose); + }).wireTap("direct:start-batch-process-csv").to("direct:pollingOutput"); from("direct:validate-tenant") .id("direct:validate-tenant") @@ -142,13 +130,11 @@ private void setup() { .log("Completed route direct:validate-tenant"); // this route is responsible for editing the incoming records based on configuration - // this step is done to make sure the file format of CSV is not altered and only the data is updated based on config - from("direct:update-incoming-data") - .id("direct:update-incoming-data") - .log("direct:update-incoming-data") + // this step is done to make sure the file format of CSV is not altered and only the data is updated based on + // config + from("direct:update-incoming-data").id("direct:update-incoming-data").log("direct:update-incoming-data") // [LOCAL_FILE_PATH] is already set in [direct:validateFileSyncResponse] route - .setProperty(LOCAL_FILE_PATH, exchangeProperty(FILE_NAME)) - .to("direct:get-transaction-array") + .setProperty(LOCAL_FILE_PATH, exchangeProperty(FILE_NAME)).to("direct:get-transaction-array") // make sure new data is set under the exchange variable [RESULT_TRANSACTION_LIST] .process(exchange -> { String registeringInstituteId = exchange.getProperty(REGISTERING_INSTITUTE_ID, String.class); @@ -157,6 +143,7 @@ private void setup() { if (!(StringUtils.hasText(registeringInstituteId) && StringUtils.hasText(programId))) { // this will make sure the file is not updated since there is no update in data logger.debug("InstitutionId or programId is null"); + exchange.setProperty(IS_UPDATED, false); return; } @@ -167,6 +154,7 @@ private void setup() { if (registeringInstitutionConfig == null) { logger.debug("Element in nested in config: {}", budgetAccountConfig.getRegisteringInstitutions().get(0).getPrograms().size()); logger.debug("Registering institute id is null"); + exchange.setProperty(IS_UPDATED, false); return; } @@ -194,8 +182,7 @@ private void setup() { exchange.setProperty(PROGRAM_NAME, program.getName()); exchange.setProperty(PAYER_IDENTIFIER_TYPE, program.getIdentifierType()); exchange.setProperty(PAYER_IDENTIFIER_VALUE, program.getIdentifierValue()); - }) - .choice() + }).choice() // update only when previous(edit function) makes any changes to data .when(exchange -> exchange.getProperty(IS_UPDATED, Boolean.class)) // warning: changing this flag can break things @@ -204,15 +191,12 @@ private void setup() { .otherwise() .log(LoggingLevel.INFO, "No update"); - from("direct:start-batch-process-csv") - .id("direct:start-batch-process-csv") - .log("Starting route direct:start-batch-process-csv") - .to("direct:update-incoming-data") - .process(exchange -> { + from("direct:start-batch-process-csv").id("direct:start-batch-process-csv").log("Starting route direct:start-batch-process-csv") + .to("direct:update-incoming-data").process(exchange -> { String fileName = exchange.getProperty(FILE_NAME, String.class); String requestId = exchange.getProperty(REQUEST_ID, String.class); String purpose = exchange.getProperty(PURPOSE, String.class); - String batchId = exchange.getProperty(BATCH_ID,String.class); + String batchId = exchange.getProperty(BATCH_ID, String.class); String note = null; if (purpose == null || purpose.isEmpty()) { @@ -227,6 +211,7 @@ private void setup() { file.setReadable(true); logger.debug("File absolute path: {}", file.getAbsolutePath()); + boolean verifyData = verifyData(file); logger.debug("Data verification result {}", verifyData); if (!verifyData) { @@ -237,9 +222,9 @@ private void setup() { logger.debug("File uploaded {}", nm); - //extracting and setting callback Url + // extracting and setting callback Url String callbackUrl = exchange.getIn().getHeader("X-Callback-URL", String.class); - exchange.setProperty(CALLBACK_URL,callbackUrl); + exchange.setProperty(CALLBACK_URL, callbackUrl); List phases = phaseUtils.getValues(); logger.debug(phases.toString()); @@ -267,8 +252,7 @@ private void setup() { JSONObject response = new JSONObject(); try { - String tenantSpecificWorkflowId = workflowId.replace("{dfspid}", - exchange.getProperty(TENANT_NAME).toString()); + String tenantSpecificWorkflowId = workflowId.replace("{dfspid}", exchange.getProperty(TENANT_NAME).toString()); String txnId = zeebeProcessStarter.startZeebeWorkflow(tenantSpecificWorkflowId, "", variables); if (txnId == null || txnId.isEmpty()) { response.put("errorCode", 500); @@ -287,25 +271,18 @@ private void setup() { exchange.getIn().setBody(response.toString()); - }) - .log("Completed route direct:start-batch-process-csv"); + }).log("Completed route direct:start-batch-process-csv"); - from("direct:start-batch-process-raw") - .id("direct:start-batch-process-raw") - .log("Starting route direct:start-batch-process-raw") + from("direct:start-batch-process-raw").id("direct:start-batch-process-raw").log("Starting route direct:start-batch-process-raw") .process(exchange -> { JSONObject response = new JSONObject(); response.put("batch_id", UUID.randomUUID().toString()); response.put("request_id", UUID.randomUUID().toString()); response.put("status", "queued"); exchange.getIn().setBody(response.toString()); - }) - .log("Completed route direct:start-batch-process-raw"); + }).log("Completed route direct:start-batch-process-raw"); - from("direct:executeBatch") - .id("direct:executeBatch") - .log("Starting route direct:executeBatch") - .to("direct:validate-tenant") + from("direct:executeBatch").id("direct:executeBatch").log("Starting route direct:executeBatch").to("direct:validate-tenant") .process(exchange -> { String filename = exchange.getIn().getHeader("filename", String.class); String requestId = exchange.getIn().getHeader("X-CorrelationID", String.class); @@ -326,8 +303,7 @@ private void setup() { .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("raw")) .to("direct:start-batch-process-raw") .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("csv")) - .to("direct:start-batch-process-csv") - .otherwise() + .to("direct:start-batch-process-csv").otherwise() .setBody(exchange -> getUnsupportedTypeJson(exchange.getProperty(BATCH_REQUEST_TYPE, String.class)).toString()) .log("Completed execution of route rest:POST:/batchtransactions"); @@ -336,10 +312,9 @@ private void setup() { json.put("PollingPath", "/batch/Summary/" + exchange.getProperty(BATCH_ID)); json.put("SuggestedCallbackSeconds", pollApiTimer); exchange.getIn().setBody(json.toString()); - exchange.getIn().setHeader(Exchange.HTTP_RESPONSE_CODE, 202); + exchange.getIn().setHeader(Exchange.HTTP_RESPONSE_CODE, 202); }); - from("direct:validateFileSyncResponse").id("direct:validateFileSyncResponse").log("Starting route direct:validateFileSyncResponse") .process(exchange -> { // move this logic to spring @@ -470,9 +445,9 @@ private Map setConfigProperties(Map variables) { variables.put(MAX_STATUS_RETRY, maxThresholdCheckRetry); variables.put(COMPLETION_THRESHOLD, completionThreshold); variables.put(THRESHOLD_DELAY, Utils.getZeebeTimerValue(thresholdCheckDelay)); - variables.put(BULK_NOTIF_SUCCESS,false); - variables.put(BULK_NOTIF_FAILURE,false); - variables.put(MAX_CALLBACK_RETRY,maxCallbackRetry); + variables.put(BULK_NOTIF_SUCCESS, false); + variables.put(BULK_NOTIF_FAILURE, false); + variables.put(MAX_CALLBACK_RETRY, maxCallbackRetry); return variables; } diff --git a/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java b/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java index 421ba7cc..43940886 100644 --- a/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java +++ b/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java @@ -72,8 +72,8 @@ public InputStream streamFile(String fileName, String bucketName) { try { File temp = new File("/temp/" + fileName); BlobInputStream csvInputStream = client.containerName(bucketName).blobName(fileName).buildClient().openInputStream(); -// byte[] content = Files.Paths.get(temp.getPath())); -// temp.delete(); + // byte[] content = Files.Paths.get(temp.getPath())); + // temp.delete(); return csvInputStream; } catch (Exception e) { e.printStackTrace(); diff --git a/src/main/java/org/mifos/processor/bulk/file/FileStorageService.java b/src/main/java/org/mifos/processor/bulk/file/FileStorageService.java index 198c0a4e..42967c3c 100644 --- a/src/main/java/org/mifos/processor/bulk/file/FileStorageService.java +++ b/src/main/java/org/mifos/processor/bulk/file/FileStorageService.java @@ -3,9 +3,15 @@ import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; +import java.io.InputStream; + @Service public interface FileStorageService { public String save(MultipartFile file); + String save(InputStream file, String filename); + + String save(String data, String filename); + } diff --git a/src/main/java/org/mifos/processor/bulk/file/FileStorageServiceImpl.java b/src/main/java/org/mifos/processor/bulk/file/FileStorageServiceImpl.java index a83e0f61..bc7ad934 100644 --- a/src/main/java/org/mifos/processor/bulk/file/FileStorageServiceImpl.java +++ b/src/main/java/org/mifos/processor/bulk/file/FileStorageServiceImpl.java @@ -2,6 +2,9 @@ import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -23,4 +26,30 @@ public String save(MultipartFile file) { } return filename; } + + @Override + public String save(InputStream inputStream, String filename) { + String uniqueFileName = getUniqueFileName(filename); + try { + Files.copy(inputStream, this.root.resolve(uniqueFileName)); + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + return uniqueFileName; + } + + @Override + public String save(String data, String filename) { + String uniqueFileName = getUniqueFileName(filename); + try { + Files.writeString(this.root.resolve(uniqueFileName), data); + } catch (IOException e) { + throw new RuntimeException(e); + } + return uniqueFileName; + } + + private String getUniqueFileName(String filename) { + return UUID.randomUUID() + "_" + Objects.requireNonNull(filename); + } } diff --git a/src/main/java/org/mifos/processor/bulk/format/EntityMapper.java b/src/main/java/org/mifos/processor/bulk/format/EntityMapper.java new file mode 100644 index 00000000..61eb1dc6 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/format/EntityMapper.java @@ -0,0 +1,14 @@ +package org.mifos.processor.bulk.format; + +import java.util.List; + +public interface EntityMapper { + + T convertTo(K object); + + K convertFrom(T object); + + List convertListTo(List objects); + + List convertListFrom(List objects); +} diff --git a/src/main/java/org/mifos/processor/bulk/format/RestRequestConvertor.java b/src/main/java/org/mifos/processor/bulk/format/RestRequestConvertor.java new file mode 100644 index 00000000..cef808ad --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/format/RestRequestConvertor.java @@ -0,0 +1,78 @@ +package org.mifos.processor.bulk.format; + +import org.mifos.processor.bulk.schema.BatchRequestDTO; +import org.mifos.processor.bulk.schema.Party; +import org.mifos.processor.bulk.schema.PartyType; +import org.mifos.processor.bulk.schema.Transaction; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +@Component +public class RestRequestConvertor implements EntityMapper { + + @Override + public BatchRequestDTO convertTo(Transaction object) { + Party creditParty = new Party(); + creditParty.setKey(PartyType.fromValue(object.getPayeeIdentifierType()).getValue()); + creditParty.setValue(object.getPayeeIdentifier()); + + Party debitParty = new Party(); + debitParty.setKey(PartyType.fromValue(object.getPayerIdentifierType()).getValue()); + debitParty.setValue(object.getPayerIdentifier()); + + // creating the DTO + BatchRequestDTO batchRequestDTO = new BatchRequestDTO(); + batchRequestDTO.setAmount(object.getAmount()); + batchRequestDTO.setCurrency(object.getCurrency()); + batchRequestDTO.setCreditParty(List.of(creditParty)); + batchRequestDTO.setDebitParty(List.of(debitParty)); + batchRequestDTO.setDescriptionText(object.getNote()); + batchRequestDTO.setSubType(object.getPaymentMode()); + + return batchRequestDTO; + } + + @Override + public Transaction convertFrom(BatchRequestDTO object) { + + // creating the transaction + Transaction transaction = new Transaction(); + transaction.setCurrency(object.getCurrency()); + transaction.setAmount(object.getAmount()); + if (object.getDebitParty() != null && object.getDebitParty().size() > 0) { + transaction.setPayerIdentifierType(object.getDebitParty().get(0).getKey()); + transaction.setPayerIdentifier(object.getDebitParty().get(0).getValue()); + } + if (object.getCreditParty() != null && object.getCreditParty().size() > 0) { + transaction.setPayeeIdentifierType(object.getCreditParty().get(0).getKey()); + transaction.setPayeeIdentifier(object.getCreditParty().get(0).getValue()); + } + transaction.setNote(object.getDescriptionText()); + transaction.setPaymentMode(object.getSubType()); + + return transaction; + } + + @Override + public List convertListTo(List objects) { + List batchRequestDTOList = new ArrayList<>(); + objects.forEach(e -> batchRequestDTOList.add(convertTo(e))); + return batchRequestDTOList; + } + + @Override + public List convertListFrom(List objects) { + List transactionList = new ArrayList<>(); + for (int i = 0; i < objects.size(); i++) { + Transaction transaction = convertFrom(objects.get(i)); + transaction.setId(i); + transaction.setRequestId(UUID.randomUUID().toString()); + transactionList.add(transaction); + } + return transactionList; + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/schema/BatchRequestDTO.java b/src/main/java/org/mifos/processor/bulk/schema/BatchRequestDTO.java new file mode 100644 index 00000000..c0ce1cc1 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/BatchRequestDTO.java @@ -0,0 +1,20 @@ +package org.mifos.processor.bulk.schema; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.util.List; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class BatchRequestDTO { + + List creditParty, debitParty; + + String subType, amount, currency, descriptionText; + +} diff --git a/src/main/java/org/mifos/processor/bulk/schema/CamelApiResponse.java b/src/main/java/org/mifos/processor/bulk/schema/CamelApiResponse.java new file mode 100644 index 00000000..4fe28952 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/CamelApiResponse.java @@ -0,0 +1,14 @@ +package org.mifos.processor.bulk.schema; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +public class CamelApiResponse { + + String body; + int status; +} diff --git a/src/main/java/org/mifos/processor/bulk/schema/Party.java b/src/main/java/org/mifos/processor/bulk/schema/Party.java new file mode 100644 index 00000000..92fdb5a1 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/Party.java @@ -0,0 +1,16 @@ +package org.mifos.processor.bulk.schema; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class Party { + + String key, value; + +} diff --git a/src/main/java/org/mifos/processor/bulk/schema/PartyType.java b/src/main/java/org/mifos/processor/bulk/schema/PartyType.java new file mode 100644 index 00000000..36fbe060 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/PartyType.java @@ -0,0 +1,26 @@ +package org.mifos.processor.bulk.schema; + +import java.util.Arrays; + +public enum PartyType { + + MSISDN("msisdn"), ACCOUNT_NUMBER("accountnumber"), EMPTY(""); + + private String value; + + PartyType(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public static PartyType fromValue(String value) { + return Arrays.stream(values()).filter(ec -> ec.getValue().equalsIgnoreCase(value)).findFirst().orElseGet(() -> PartyType.EMPTY); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/utility/CsvWriter.java b/src/main/java/org/mifos/processor/bulk/utility/CsvWriter.java new file mode 100644 index 00000000..95694fce --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/utility/CsvWriter.java @@ -0,0 +1,32 @@ +package org.mifos.processor.bulk.utility; + +import com.fasterxml.jackson.databind.SequenceWriter; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +public class CsvWriter { + + public static void writeToCsv(List data, Class tClass, CsvMapper csvMapper, boolean overrideHeader, String filepath) + throws IOException { + CsvSchema csvSchema = csvMapper.schemaFor(tClass); + if (overrideHeader) { + csvSchema = csvSchema.withHeader(); + } else { + csvSchema = csvSchema.withoutHeader(); + } + + File file = new File(filepath); + if (!file.exists()) { + file.createNewFile(); + } + SequenceWriter writer = csvMapper.writerWithSchemaFor(tClass).with(csvSchema).writeValues(file); + for (T object : data) { + writer.write(object); + } + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 81337eae..1965f6f8 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -129,4 +129,7 @@ private ZeebeVariables() {} public static final String PROGRAM_NAME = "programName"; public static final String PAYER_IDENTIFIER_TYPE = "payerIdentifierType"; public static final String PAYER_IDENTIFIER_VALUE = "payerIdentifier"; + public static final String HEADER_CLIENT_CORRELATION_ID = "X-CorrelationID"; + public static final String HEADER_TYPE = "Type"; + public static final String HEADER_PLATFORM_TENANT_ID = "Platform-TenantId"; } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 214d5ef5..6ea1c4ed 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -142,7 +142,7 @@ bulk-processor: hostname : "https://bulk-connector.sandbox.mifos.io" csv: - columnNames: "id,request_id,payment_mode,payer_identifier_type,payer_identifier,payee_identifier_type,payee_identifier,amount,currency,note" + columnNames: "id,request_id,payment_mode,payer_identifier_type,payer_identifier,payee_identifier_type,payee_identifier,amount,currency,note,account_number,program_shortcode,cycle" size : 100000 # in bytes budget-account: From 814435834d912b7575de8f3d3a4c9c47c38eb733 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Mon, 4 Sep 2023 18:28:48 +0530 Subject: [PATCH 092/156] PHEE-420 Updated the connector-common version (#111) --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 5e39a38e..e1b50959 100644 --- a/build.gradle +++ b/build.gradle @@ -108,7 +108,7 @@ dependencies { // miscellaneous dependency implementation 'com.google.code.gson:gson:2.8.9' implementation 'org.json:json:20210307' - implementation 'org.mifos:ph-ee-connector-common:1.6.1-rc.1' + implementation 'org.mifos:ph-ee-connector-common:1.6.1-rc.2' implementation 'org.apache.camel.springboot:camel-spring-boot-starter:3.4.0' implementation 'org.apache.camel:camel-undertow:3.4.0' implementation 'org.springframework.boot:spring-boot-starter:2.5.2' From b68587a01a3481d8dc4597f3ff55cf0926722b5a Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Thu, 7 Sep 2023 18:41:07 +0530 Subject: [PATCH 093/156] Channle sevice i https hence making default hostname as https (#113) --- src/main/resources/application.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 6ea1c4ed..aebf3181 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -38,7 +38,7 @@ operations-app: batch-transaction: "/api/v1/batch/transactions" channel: - hostname: "http://ph-ee-connector-channel" + hostname: "https://ph-ee-connector-channel" cloud: aws: From f08953bec53c3343a2643f90e405af6adf9fd86e Mon Sep 17 00:00:00 2001 From: Manoj <56669674+fynmanoj@users.noreply.github.com> Date: Fri, 8 Sep 2023 00:07:16 +0530 Subject: [PATCH 094/156] updated-connector-common-version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index e1b50959..59f43ed6 100644 --- a/build.gradle +++ b/build.gradle @@ -108,7 +108,7 @@ dependencies { // miscellaneous dependency implementation 'com.google.code.gson:gson:2.8.9' implementation 'org.json:json:20210307' - implementation 'org.mifos:ph-ee-connector-common:1.6.1-rc.2' + implementation 'org.mifos:ph-ee-connector-common:1.6.0' implementation 'org.apache.camel.springboot:camel-spring-boot-starter:3.4.0' implementation 'org.apache.camel:camel-undertow:3.4.0' implementation 'org.springframework.boot:spring-boot-starter:2.5.2' From b5db18d8eed3fd77232da33c890e3173a4207190 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Tue, 26 Sep 2023 16:59:36 +0530 Subject: [PATCH 095/156] Fixed the channel connector hostname (#117) --- src/main/resources/application.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index aebf3181..3576a211 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -38,7 +38,7 @@ operations-app: batch-transaction: "/api/v1/batch/transactions" channel: - hostname: "https://ph-ee-connector-channel" + hostname: "https://ph-ee-connector-channel:8443" cloud: aws: From 604829aa229dc63db45d135bbd01c602fb0acdc8 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Thu, 12 Oct 2023 18:22:36 +0530 Subject: [PATCH 096/156] GOV-359: Add batch aggregate worker to bulk processor (#118) * remove unnecessary merge conflict text from application.yaml (#88) GOV-359: code changes for updated bpmn GOV-359: code refactor GOV-359: fix checkstyle errors GOV-359: change class names GOV-359: update route id GOV-359: fix route naming GOV-359: fix issue with multiple route with same name GOV-359: update batch aggregate response variable name GOV-359: address review comments GOV-359: address review comments GOV-359: use hyphen separated for camel case * Added config and if else logic for batch status and agregate worker * Added log statement in worker * Fixed the batch aggregate external api call * Synced the BatchDTO with latest one from ops-ap * Fixed the BatchDTO * Changed the threshold timer --------- Co-authored-by: Arka Das <127482132+arkadasfynarfin@users.noreply.github.com> --- .../processor/bulk/OperationsAppConfig.java | 6 + ...tusRoute.java => BatchAggregateRoute.java} | 40 ++-- .../camel/routes/ProcessorStartRoute.java | 1 + .../processor/bulk/camel/routes/RouteId.java | 7 +- .../mifos/processor/bulk/kafka/Consumers.java | 2 +- .../mifos/processor/bulk/schema/BatchDTO.java | 182 ++---------------- .../processor/bulk/zeebe/ZeebeVariables.java | 2 + .../bulk/zeebe/worker/AggregateWorker.java | 55 ++++++ .../bulk/zeebe/worker/BatchStatusWorker.java | 88 +++++---- .../bulk/zeebe/worker/InitSubBatchWorker.java | 2 + .../processor/bulk/zeebe/worker/Worker.java | 8 +- .../bulk/zeebe/worker/WorkerConfig.java | 3 + src/main/resources/application.yaml | 5 +- 13 files changed, 175 insertions(+), 226 deletions(-) rename src/main/java/org/mifos/processor/bulk/camel/routes/{BatchStatusRoute.java => BatchAggregateRoute.java} (59%) create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java diff --git a/src/main/java/org/mifos/processor/bulk/OperationsAppConfig.java b/src/main/java/org/mifos/processor/bulk/OperationsAppConfig.java index 78b52478..4c754802 100644 --- a/src/main/java/org/mifos/processor/bulk/OperationsAppConfig.java +++ b/src/main/java/org/mifos/processor/bulk/OperationsAppConfig.java @@ -16,6 +16,9 @@ public class OperationsAppConfig { @Value("${operations-app.endpoints.batch-summary}") public String batchSummaryEndpoint; + @Value("${operations-app.endpoints.batch-aggregate}") + public String batchAggregateEndpoint; + @Value("${operations-app.endpoints.auth}") public String authEndpoint; @@ -29,6 +32,8 @@ public class OperationsAppConfig { public String batchSummaryUrl; + public String batchAggregateUrl; + public String authUrl; @PostConstruct @@ -36,5 +41,6 @@ private void setup() { batchTransactionUrl = operationAppContactPoint + batchTransactionEndpoint; batchSummaryUrl = operationAppContactPoint + batchSummaryEndpoint; authUrl = operationAppContactPoint + authEndpoint; + batchAggregateUrl = operationAppContactPoint + batchAggregateEndpoint; } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java similarity index 59% rename from src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java rename to src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java index 3b335fdb..b51586eb 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchStatusRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java @@ -15,7 +15,7 @@ import org.springframework.stereotype.Component; @Component -public class BatchStatusRoute extends BaseRouteBuilder { +public class BatchAggregateRoute extends BaseRouteBuilder { @Value("${config.completion-threshold-check.completion-threshold}") private int completionThreshold; @@ -23,7 +23,7 @@ public class BatchStatusRoute extends BaseRouteBuilder { @Override public void configure() throws Exception { - from("rest:get:test/batch/summary").to(RouteId.BATCH_STATUS.getValue()); + from("rest:get:test/batch/summary").to(RouteId.BATCH_AGGREGATE.getValue()); /** * Base route for kicking off ordering logic. Performs below tasks. 1. Downloads the csv form cloud. 2. Builds @@ -31,27 +31,35 @@ public void configure() throws Exception { * configuration provided in application.yaml. @see [Standard.java] 4. Update file with the updated data. 5. * Uploads the updated file in cloud. */ - from(RouteId.BATCH_STATUS.getValue()).id(RouteId.BATCH_STATUS.getValue()).log("Starting route " + RouteId.BATCH_STATUS.name()) - .to("direct:get-access-token").choice().when(exchange -> exchange.getProperty(OPS_APP_ACCESS_TOKEN, String.class) != null) - .log(LoggingLevel.INFO, "Got access token, moving on to API call").to("direct:batch-summary") - .to("direct:batch-summary-response-handler").otherwise().log(LoggingLevel.INFO, "Authentication failed.").endChoice(); + from(RouteId.BATCH_AGGREGATE.getValue()).id(RouteId.BATCH_AGGREGATE.getValue()) + .log("Starting route " + RouteId.BATCH_AGGREGATE.name()) + .to("direct:get-access-token") + .choice() + .when(exchange -> exchange.getProperty(OPS_APP_ACCESS_TOKEN, String.class) != null) + .log(LoggingLevel.INFO, "Got access token, moving on to API call") + .to("direct:batch-aggregate-api-call") + .to("direct:batch-aggregate-response-handler") + .otherwise() + .log(LoggingLevel.INFO, "Authentication failed.") + .endChoice(); - getBaseExternalApiRequestRouteDefinition("batch-summary", HttpRequestMethod.GET) - .setHeader(Exchange.REST_HTTP_QUERY, simple("batchId=${exchangeProperty." + BATCH_ID + "}")) + getBaseExternalApiRequestRouteDefinition("batch-aggregate-api-call", HttpRequestMethod.GET) +// .setHeader(Exchange.REST_HTTP_QUERY, simple("batchId=${exchangeProperty." + BATCH_ID + "}")) .setHeader("Authorization", simple("Bearer ${exchangeProperty." + OPS_APP_ACCESS_TOKEN + "}")) .setHeader(HEADER_PLATFORM_TENANT_ID, simple("${exchangeProperty." + TENANT_ID + "}")).process(exchange -> { logger.info(exchange.getIn().getHeaders().toString()); - }).toD(operationsAppConfig.batchSummaryUrl + "?bridgeEndpoint=true") - .log(LoggingLevel.INFO, "Batch summary API response: \n\n ${body}"); + }).toD(operationsAppConfig.batchAggregateUrl + "${exchangeProperty." + BATCH_ID + "}?bridgeEndpoint=true") + .log(LoggingLevel.DEBUG, "Batch aggregate API response: \n\n ${body}") + .log(LoggingLevel.INFO, "Aggregate Response body: ${body}"); - from("direct:batch-summary-response-handler").id("direct:batch-summary-response-handler") - .log("Starting route direct:batch-summary-response-handler") + from("direct:batch-aggregate-response-handler") + .id("direct:batch-aggregate-response-handler") + .log("Starting route direct:batch-aggregate-response-handler") // .setBody(exchange -> exchange.getIn().getBody(String.class)) .choice().when(header("CamelHttpResponseCode").isEqualTo("200")).log(LoggingLevel.INFO, "Batch summary request successful") .unmarshal().json(JsonLibrary.Jackson, BatchDTO.class).process(exchange -> { - BatchDTO batchSummary = exchange.getIn().getBody(BatchDTO.class); - - int percentage = (int) (((double) batchSummary.getSuccessful() / batchSummary.getTotal()) * 100); + BatchDTO batchAggregateResponse = exchange.getIn().getBody(BatchDTO.class); + int percentage = (int) (((double) batchAggregateResponse.getSuccessful() / batchAggregateResponse.getTotal()) * 100); if (percentage >= completionThreshold) { logger.info("Batch success threshold reached. Expected rate: {}, Actual Rate: {}", completionThreshold, percentage); @@ -59,7 +67,7 @@ public void configure() throws Exception { exchange.setProperty(COMPLETION_RATE, percentage); - }).otherwise().log(LoggingLevel.ERROR, "Batch summary request unsuccessful").process(exchange -> { + }).otherwise().log(LoggingLevel.ERROR, "Batch aggregate request unsuccessful").process(exchange -> { exchange.setProperty(BATCH_STATUS_FAILED, true); exchange.setProperty(ERROR_DESCRIPTION, exchange.getIn().getBody(String.class)); exchange.setProperty(ERROR_CODE, exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE)); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index dee022ee..3a089b27 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -435,6 +435,7 @@ private void setResponse(Exchange exchange, int responseCode) { } private Map setConfigProperties(Map variables) { + variables.put(BATCH_AGGREGATE_ENABLED, workerConfig.isBatchAggregateEnabled); variables.put(PARTY_LOOKUP_ENABLED, workerConfig.isPartyLookUpWorkerEnabled); variables.put(APPROVAL_ENABLED, workerConfig.isApprovalWorkerEnabled); variables.put(ORDERING_ENABLED, workerConfig.isOrderingWorkerEnabled); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java b/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java index 9af8fae8..b4bede34 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java @@ -2,9 +2,10 @@ public enum RouteId { - PARTY_LOOKUP("direct:partyLookup"), APPROVAL("direct:approval"), ORDERING("direct:ordering"), SPLITTING("direct:splitting"), FORMATTING( - "direct:formatting"), BATCH_STATUS("direct:batchStatus"), SEND_CALLBACK( - "direct:sendCallback"), MERGE_BACK("direct:mergeSubBatch"), INIT_SUB_BATCH("direct:init-sub-batches"); + PARTY_LOOKUP("direct:partyLookup"), APPROVAL("direct:approval"), ORDERING("direct:ordering"), + SPLITTING("direct:splitting"), FORMATTING("direct:formatting"), BATCH_STATUS("direct:batchStatus"), + SEND_CALLBACK("direct:sendCallback"), MERGE_BACK("direct:mergeSubBatch"), + INIT_SUB_BATCH("direct:init-sub-batches"), BATCH_AGGREGATE("direct:batch-aggregate"); private final String value; diff --git a/src/main/java/org/mifos/processor/bulk/kafka/Consumers.java b/src/main/java/org/mifos/processor/bulk/kafka/Consumers.java index 09e64067..03769b77 100644 --- a/src/main/java/org/mifos/processor/bulk/kafka/Consumers.java +++ b/src/main/java/org/mifos/processor/bulk/kafka/Consumers.java @@ -49,7 +49,7 @@ public class Consumers { @KafkaListener(topics = "${kafka.topic.gsma.name}", groupId = "group_id") public void listenTopicGsma(String message) throws JsonProcessingException { log.debug("Received Message in topic GSMA and group group_id: {}", message); - TransactionOlder transaction = objectMapper.readValue((String) message, TransactionOlder.class); + TransactionOlder transaction = objectMapper.readValue(message, TransactionOlder.class); String tenantId = "ibank-usa"; GSMATransaction gsmaChannelRequest = new GSMATransaction(); diff --git a/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java b/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java index 26a25536..b98386cf 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java +++ b/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java @@ -1,12 +1,21 @@ package org.mifos.processor.bulk.schema; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + import java.math.BigDecimal; +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor public class BatchDTO { - private String batchId; + private String batch_id; - private String requestId; + private String request_id; private Long total; @@ -28,7 +37,7 @@ public class BatchDTO { private String notes; - private String createdAt; + private String created_at; private String status; @@ -36,171 +45,14 @@ public class BatchDTO { private String purpose; - public BatchDTO() {} - - public BatchDTO(String batchId, String requestId, Long totalTransactions, Long ongoing, Long failed, Long completed, - BigDecimal total_amount, BigDecimal completed_amount, BigDecimal ongoing_amount, BigDecimal failed_amount, String result_file, - String note) { - this.batchId = batchId; - this.requestId = requestId; - this.total = totalTransactions; - this.ongoing = ongoing; - this.failed = failed; - this.successful = completed; - this.totalAmount = total_amount; - this.successfulAmount = completed_amount; - this.pendingAmount = ongoing_amount; - this.failedAmount = failed_amount; - this.file = result_file; - this.notes = note; - } - - public BatchDTO(String batch_id, String request_id, Long total, Long ongoing, Long failed, Long successful, BigDecimal totalAmount, - BigDecimal successfulAmount, BigDecimal pendingAmount, BigDecimal failedAmount, String file, String notes, String created_at, - String status, String modes, String purpose) { - this.batchId = batch_id; - this.requestId = request_id; - this.total = total; - this.ongoing = ongoing; - this.failed = failed; - this.successful = successful; - this.totalAmount = totalAmount; - this.successfulAmount = successfulAmount; - this.pendingAmount = pendingAmount; - this.failedAmount = failedAmount; - this.file = file; - this.notes = notes; - this.createdAt = created_at; - this.status = status; - this.modes = modes; - this.purpose = purpose; - } - - public String getBatch_id() { - return batchId; - } - - public void setBatch_id(String batch_id) { - this.batchId = batch_id; - } - - public String getRequest_id() { - return requestId; - } - - public void setRequest_id(String request_id) { - this.requestId = request_id; - } - - public Long getTotal() { - return total; - } - - public void setTotal(Long total) { - this.total = total; - } - - public Long getOngoing() { - return ongoing; - } - - public void setOngoing(Long ongoing) { - this.ongoing = ongoing; - } - - public Long getFailed() { - return failed; - } - - public void setFailed(Long failed) { - this.failed = failed; - } - - public Long getSuccessful() { - return successful; - } - - public void setSuccessful(Long successful) { - this.successful = successful; - } - - public BigDecimal getTotalAmount() { - return totalAmount; - } - - public void setTotalAmount(BigDecimal totalAmount) { - this.totalAmount = totalAmount; - } - - public BigDecimal getSuccessfulAmount() { - return successfulAmount; - } - - public void setSuccessfulAmount(BigDecimal successfulAmount) { - this.successfulAmount = successfulAmount; - } - - public BigDecimal getPendingAmount() { - return pendingAmount; - } - - public void setPendingAmount(BigDecimal pendingAmount) { - this.pendingAmount = pendingAmount; - } - - public BigDecimal getFailedAmount() { - return failedAmount; - } - - public void setFailedAmount(BigDecimal failedAmount) { - this.failedAmount = failedAmount; - } - - public String getFile() { - return file; - } - - public void setFile(String file) { - this.file = file; - } - - public String getNotes() { - return notes; - } - - public void setNotes(String notes) { - this.notes = notes; - } - - public String getCreated_at() { - return createdAt; - } - - public void setCreated_at(String created_at) { - this.createdAt = created_at; - } - - public String getStatus() { - return status; - } + private String failPercentage; - public void setStatus(String status) { - this.status = status; - } + private String successPercentage; - public String getModes() { - return modes; - } + private String registeringInstitutionId; - public void setModes(String modes) { - this.modes = modes; - } + private String payerFsp; - public String getPurpose() { - return purpose; - } + private String correlationId; - public void setPurpose(String purpose) { - this.purpose = purpose; - } } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 1965f6f8..0cbc5a96 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -54,6 +54,8 @@ private ZeebeVariables() {} public static final String FORMATTING_ENABLED = "formattingEnabled"; + public static final String BATCH_AGGREGATE_ENABLED = "batchAggregateEnabled"; + public static final String COMPLETION_THRESHOLD_CHECK_ENABLED = "completionThresholdCheckEnabled"; public static final String MERGE_ENABLED = "mergeEnabled"; diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java new file mode 100644 index 00000000..5f437f3e --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java @@ -0,0 +1,55 @@ +package org.mifos.processor.bulk.zeebe.worker; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_STATUS_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_RATE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_CODE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_DESCRIPTION; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.RETRY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; + +import java.util.Map; +import org.apache.camel.Exchange; +import org.apache.camel.support.DefaultExchange; +import org.mifos.processor.bulk.camel.routes.RouteId; +import org.springframework.stereotype.Component; + +@Component +public class AggregateWorker extends BaseWorker { + + @Override + public void setup() { +// newWorker(Worker.BATCH_STATUS, (client, job) -> { + newWorker(Worker.BATCH_AGGREGATE, (client, job) -> { + logger.info("Started batchAggregateWorker"); + logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); + Map variables = job.getVariablesAsMap(); + + int retry = (int) variables.getOrDefault(RETRY, 0); + int successRate = (int) variables.getOrDefault(COMPLETION_RATE, 0); + + Exchange exchange = new DefaultExchange(camelContext); + exchange.setProperty(BATCH_ID, variables.get(BATCH_ID)); + exchange.setProperty(TENANT_ID, variables.get(TENANT_ID)); + + sendToCamelRoute(RouteId.BATCH_AGGREGATE, exchange); + + Boolean batchStatusFailed = exchange.getProperty(BATCH_STATUS_FAILED, Boolean.class); + if (batchStatusFailed == null || !batchStatusFailed) { + successRate = exchange.getProperty(COMPLETION_RATE, Long.class).intValue(); + } else { + variables.put(ERROR_CODE, exchange.getProperty(ERROR_CODE)); + variables.put(ERROR_DESCRIPTION, exchange.getProperty(ERROR_DESCRIPTION)); + logger.info("Error: {}, {}", variables.get(ERROR_CODE), variables.get(ERROR_DESCRIPTION)); + } + + variables.put(COMPLETION_RATE, successRate); + variables.put(RETRY, ++retry); + + logger.info("Retry: {} and Success Rate: {}", retry, successRate); + client.newCompleteCommand(job.getKey()).variables(variables).send(); + logger.info("Completed batchAggregateWorker"); + }); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java index e2edc0c4..0114e0b4 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java @@ -1,52 +1,68 @@ package org.mifos.processor.bulk.zeebe.worker; -import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_STATUS_FAILED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_RATE; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_CODE; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_DESCRIPTION; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.RETRY; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.mifos.processor.bulk.OperationsAppConfig; +import org.mifos.processor.bulk.schema.BatchDTO; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.*; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; import java.util.Map; -import org.apache.camel.Exchange; -import org.apache.camel.support.DefaultExchange; -import org.mifos.processor.bulk.camel.routes.RouteId; -import org.springframework.stereotype.Component; -@Component -public class BatchStatusWorker extends BaseWorker { +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; + +public class BatchStatusWorker extends BaseWorker{ + + @Autowired + public OperationsAppConfig operationsAppConfig; @Override public void setup() { - newWorker(Worker.BATCH_STATUS, (client, job) -> { - logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); - Map variables = job.getVariablesAsMap(); - int retry = (int) variables.getOrDefault(RETRY, 0); - int successRate = (int) variables.getOrDefault(COMPLETION_RATE, 0); + newWorker(Worker.BATCH_STATUS, ((client, job) -> { + logger.info("Started batchStatusWorker"); + logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); - Exchange exchange = new DefaultExchange(camelContext); - exchange.setProperty(BATCH_ID, variables.get(BATCH_ID)); - exchange.setProperty(TENANT_ID, variables.get(TENANT_ID)); + Map variables = job.getVariablesAsMap(); + String batchId = (String) variables.get(BATCH_ID); + String tenantId = (String) variables.get(TENANT_ID); + BatchDTO batchDTOResponse = invokeBatchAggregationApi(batchId, tenantId); + float successRate = calculateSuccessPercentage(batchDTOResponse); + variables.put(COMPLETION_RATE, successRate); + client.newCompleteCommand(job.getKey()).variables(variables).send(); + logger.info("Completed batchStatusWorker"); + })); + } - sendToCamelRoute(RouteId.BATCH_STATUS, exchange); + private float calculateSuccessPercentage(BatchDTO batchDTO){ + if(batchDTO.getTotal()!=null && batchDTO.getTotal()!=0){ + return (((float) batchDTO.getSuccessful() / batchDTO.getTotal()) * 100); + } + return 0; + } - Boolean batchStatusFailed = exchange.getProperty(BATCH_STATUS_FAILED, Boolean.class); - if (batchStatusFailed == null || !batchStatusFailed) { - successRate = exchange.getProperty(COMPLETION_RATE, Integer.class); - } else { - variables.put(ERROR_CODE, exchange.getProperty(ERROR_CODE)); - variables.put(ERROR_DESCRIPTION, exchange.getProperty(ERROR_DESCRIPTION)); - logger.info("Error: {}, {}", variables.get(ERROR_CODE), variables.get(ERROR_DESCRIPTION)); - } + private BatchDTO invokeBatchAggregationApi(String batchId, String tenantId) { + RestTemplate restTemplate = new RestTemplate(); + HttpHeaders headers = new HttpHeaders(); + headers.set("Platform-TenantId", tenantId); + String url = operationsAppConfig.batchSummaryUrl; - variables.put(COMPLETION_RATE, successRate); - variables.put(RETRY, ++retry); + UriComponentsBuilder uriBuilder = UriComponentsBuilder.fromUriString(url).queryParam("batchId", batchId); + String finalUrl = uriBuilder.toUriString(); - logger.info("Retry: {} and Success Rate: {}", retry, successRate); - client.newCompleteCommand(job.getKey()).variables(variables).send(); - }); + ResponseEntity response = restTemplate.exchange(finalUrl, HttpMethod.GET, + new HttpEntity<>(null, headers), String.class); + String batchAggregationResponse = response != null ? response.getBody() : null; + ObjectMapper objectMapper = new ObjectMapper(); + BatchDTO batchDTO = null; + try { + batchDTO = objectMapper.readValue(batchAggregationResponse, BatchDTO.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + logger.info("Batch summary response: {}", batchDTO); + return batchDTO; } - } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java index 787a8596..d206d445 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java @@ -33,6 +33,7 @@ public void setup() { * Parse the data into POJO. 3. Initiates workflow based on the payment_mode */ newWorker(Worker.INIT_SUB_BATCH, (client, job) -> { + logger.info("Started INIT_SUB_BATCH worker"); logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); Map variables = job.getVariablesAsMap(); @@ -78,6 +79,7 @@ public void setup() { variables.put(INIT_FAILURE_SUB_BATCHES, failureSubBatches); client.newCompleteCommand(job.getKey()).variables(variables).send(); + logger.info("Completed INIT_SUB_BATCH worker. Remaining subbatches {}", subBatches.size()); }); } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java index 3f2ed26d..3e3e7daa 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java @@ -2,10 +2,10 @@ public enum Worker { - PARTY_LOOKUP("partyLookup"), APPROVAL("approval"), ORDERING("ordering"), SPLITTING("splitting"), FORMATTING("formatting"), BATCH_STATUS( - "batchStatus"), SEND_CALLBACK("sendCallback"), MERGE_BACK("mergeSubBatch"), - - INIT_SUB_BATCH("initSubBatch"), ACCOUNT_LOOKUP("accountLookup"), ACCOUNT_LOOKUP_CALLBACK("accountLookupCallback"); + PARTY_LOOKUP("partyLookup"), APPROVAL("approval"), ORDERING("ordering"), SPLITTING("splitting"), + FORMATTING("formatting"), BATCH_STATUS("batchStatus"), SEND_CALLBACK("sendCallback"), + MERGE_BACK("mergeSubBatch"), INIT_SUB_BATCH("initSubBatch"), ACCOUNT_LOOKUP("accountLookup"), + ACCOUNT_LOOKUP_CALLBACK("accountLookupCallback"), BATCH_AGGREGATE("batchAggregate"); private final String value; diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java index 6fc8c42c..56575c0e 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java @@ -6,6 +6,9 @@ @Component public class WorkerConfig { + @Value("${config.batchAggregate.enable}") + public boolean isBatchAggregateEnabled; + @Value("${config.partylookup.enable}") public boolean isPartyLookUpWorkerEnabled; diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 3576a211..33e82a1b 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -36,6 +36,7 @@ operations-app: auth: "/oauth/token" batch-summary: "/api/v1/batch" batch-transaction: "/api/v1/batch/transactions" + batch-aggregate: "/api/v1/batch/" channel: hostname: "https://ph-ee-connector-channel:8443" @@ -79,6 +80,8 @@ bpmn: config: minimum-successful-tx-ratio: 0.90 + batchAggregate: + enable: true partylookup: enable: true approval: @@ -100,7 +103,7 @@ config: enable: false completion-threshold: 95 # in percentage max-retry: 4 #can be as high as 30 - delay: 60 # in seconds + delay: 2 # in seconds deduplication: enabled: true From 31cc6aa0a34228a7293d06495f455cdc3ac1a803 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:57:08 +0530 Subject: [PATCH 097/156] GOV-301: Add Authorization worker for Batch Authorization (#122) * GOV-301: add auth worker GOV-301: handle callback (incomplete) GOV-301: updating the response handling of async api call GOV-301: minor code refactor GOV-301: applying spotless changes testing changes GOV-301: updated changes GOV-301: add updated s3 details GOV-301: updated changes with refactoring GOV-301: working and refactored code GOV-301: refactored code GOV-301: add approved amount zeebe variable update url GOV-301: addressing review comments Fixed compile time error after conflict resolution * Fixed the authorization worker bean creation * Mock payment schema contact point set as service name * Callback endpoint referneced form service name --------- Co-authored-by: Arka Das --- .../bulk/api/CallbackController.java | 53 +++++++++++++ .../bulk/camel/routes/InitSubBatchRoute.java | 14 ++-- .../camel/routes/ProcessorStartRoute.java | 27 +++++++ .../bulk/schema/AuthorizationRequest.java | 21 +++++ .../bulk/schema/AuthorizationResponse.java | 19 +++++ .../processor/bulk/zeebe/ZeebeMessages.java | 2 + .../processor/bulk/zeebe/ZeebeVariables.java | 23 ++++++ .../zeebe/worker/AuthorizationWorker.java | 77 +++++++++++++++++++ .../processor/bulk/zeebe/worker/Worker.java | 3 +- .../bulk/zeebe/worker/WorkerConfig.java | 3 + src/main/resources/application.yaml | 10 +++ 11 files changed, 246 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/api/CallbackController.java create mode 100644 src/main/java/org/mifos/processor/bulk/schema/AuthorizationRequest.java create mode 100644 src/main/java/org/mifos/processor/bulk/schema/AuthorizationResponse.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/AuthorizationWorker.java diff --git a/src/main/java/org/mifos/processor/bulk/api/CallbackController.java b/src/main/java/org/mifos/processor/bulk/api/CallbackController.java new file mode 100644 index 00000000..76a2d82c --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/api/CallbackController.java @@ -0,0 +1,53 @@ +package org.mifos.processor.bulk.api; + +import io.camunda.zeebe.client.ZeebeClient; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import org.mifos.processor.bulk.schema.AuthorizationResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; + +@RestController +public class CallbackController { + + @Autowired + private ZeebeClient zeebeClient; + + protected Logger logger = LoggerFactory.getLogger(this.getClass()); + + private static final String EXPECTED_AUTH_STATUS = "Y"; + + + @PostMapping("/authorization/callback") + public ResponseEntity handleAuthorizationCallback(@RequestBody AuthorizationResponse authResponse) { + Map variables = new HashMap<>(); + + boolean isAuthorizationSuccessful = EXPECTED_AUTH_STATUS.equals(authResponse.getStatus()); + variables.put(AUTHORIZATION_SUCCESSFUL, isAuthorizationSuccessful); + variables.put(CLIENT_CORRELATION_ID, authResponse.getClientCorrelationId()); + variables.put(AUTHORIZATION_STATUS, authResponse.getStatus()); + variables.put(AUTHORIZATION_FAIL_REASON, authResponse.getReason()); + + if (!isAuthorizationSuccessful) { + variables.put(APPROVED_AMOUNT, 0); + } + + if (zeebeClient != null) { + zeebeClient.newPublishMessageCommand() + .messageName(AUTHORIZATION_RESPONSE) + .correlationKey(authResponse.getClientCorrelationId()) + .timeToLive(Duration.ofMillis(500000)) + .variables(variables).send(); + } + return ResponseEntity.ok().build(); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index 46063c88..c8bf3585 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -66,7 +66,7 @@ public class InitSubBatchRoute extends BaseRouteBuilder { private ExternalApiPayloadConfig externalApiPayloadConfig; @Value("${channel.hostname}") - private String ChannelURL; + private String channelURL; @Override public void configure() throws Exception { @@ -76,8 +76,12 @@ public void configure() throws Exception { * Builds the [Transaction] array using [direct:get-transaction-array] route. 3. Loops through each transaction * and start the respective workflow */ - from(RouteId.INIT_SUB_BATCH.getValue()).id(RouteId.INIT_SUB_BATCH.getValue()).log("Starting route " + RouteId.INIT_SUB_BATCH.name()) - .to("direct:download-file").to("direct:get-transaction-array").to("direct:start-workflow-step1"); + from(RouteId.INIT_SUB_BATCH.getValue()) + .id(RouteId.INIT_SUB_BATCH.getValue()) + .log("Starting route " + RouteId.INIT_SUB_BATCH.name()) + .to("direct:download-file") + .to("direct:get-transaction-array") + .to("direct:start-workflow-step1"); // crates the zeebe variables map and starts the workflow by calling >> direct:start-workflow-step2 from("direct:start-workflow-step1").id("direct:start-flow-step1").log("Starting route direct:start-flow-step1") @@ -197,8 +201,8 @@ public void configure() throws Exception { log.debug("Variables: {}", exchange.getProperties()); log.debug("Emergency: {}", exchange.getIn().getHeaders()); }) - .toD(ChannelURL + "${exchangeProperty.extEndpoint}" + "?bridgeEndpoint=true&throwExceptionOnFailure=false") - .log(LoggingLevel.INFO, "Response body: ${body}").otherwise().endChoice(); + .toD(channelURL + "${exchangeProperty.extEndpoint}" + "?bridgeEndpoint=true&throwExceptionOnFailure=false") + .log(LoggingLevel.DEBUG, "Response body: ${body}").otherwise().endChoice(); from("direct:validate-payment-mode").id("direct:validate-payment-mode").log("Starting route direct:validate-payment-mode") .process(exchange -> { diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 3a089b27..1c220823 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -4,6 +4,32 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_CLIENT_CORRELATION_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_REQUEST_TYPE; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.APPROVAL_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.AUTHORIZATION_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BULK_NOTIF_FAILURE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BULK_NOTIF_SUCCESS; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_URL; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CLIENT_CORRELATION_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_THRESHOLD; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_THRESHOLD_CHECK_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FORMATTING_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_CALLBACK_RETRY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_STATUS_RETRY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.NOTE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERING_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASES; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASE_COUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.THRESHOLD_DELAY; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -437,6 +463,7 @@ private void setResponse(Exchange exchange, int responseCode) { private Map setConfigProperties(Map variables) { variables.put(BATCH_AGGREGATE_ENABLED, workerConfig.isBatchAggregateEnabled); variables.put(PARTY_LOOKUP_ENABLED, workerConfig.isPartyLookUpWorkerEnabled); + variables.put(AUTHORIZATION_ENABLED, workerConfig.isAuthorizationWorkerEnabled); variables.put(APPROVAL_ENABLED, workerConfig.isApprovalWorkerEnabled); variables.put(ORDERING_ENABLED, workerConfig.isOrderingWorkerEnabled); variables.put(SPLITTING_ENABLED, workerConfig.isSplittingWorkerEnabled); diff --git a/src/main/java/org/mifos/processor/bulk/schema/AuthorizationRequest.java b/src/main/java/org/mifos/processor/bulk/schema/AuthorizationRequest.java new file mode 100644 index 00000000..5f479dc1 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/AuthorizationRequest.java @@ -0,0 +1,21 @@ +package org.mifos.processor.bulk.schema; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class AuthorizationRequest { + + private String batchId; + + private String payerIdentifier; + + private String currency; + + private String amount; +} diff --git a/src/main/java/org/mifos/processor/bulk/schema/AuthorizationResponse.java b/src/main/java/org/mifos/processor/bulk/schema/AuthorizationResponse.java new file mode 100644 index 00000000..809f9d04 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/AuthorizationResponse.java @@ -0,0 +1,19 @@ +package org.mifos.processor.bulk.schema; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class AuthorizationResponse { + + private String clientCorrelationId; + + private String status; + + private String reason; +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeMessages.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeMessages.java index a3e442cd..c4538776 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeMessages.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeMessages.java @@ -7,4 +7,6 @@ private ZeebeMessages() {} public static final String OPERATOR_MANUAL_RECOVERY = "operator-manual-recovery"; public static final String ACCOUNT_LOOKUP = "account-lookup"; + public static final String AUTHORIZATION_RESPONSE = "authorization-response"; + } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 0cbc5a96..57bd89f5 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -134,4 +134,27 @@ private ZeebeVariables() {} public static final String HEADER_CLIENT_CORRELATION_ID = "X-CorrelationID"; public static final String HEADER_TYPE = "Type"; public static final String HEADER_PLATFORM_TENANT_ID = "Platform-TenantId"; + + public static final String AUTHORIZATION_SUCCESSFUL = "authorizationSuccessful"; + + public static final String AUTHORIZATION_ACCEPTED = "authorizationAccepted"; + + public static final String APPROVED_AMOUNT = "approvedAmount"; + + public static final String AUTHORIZATION_ENABLED = "authorizationEnabled"; + + public static final String CLIENT_CORRELATION_ID = "clientCorrelationId"; + + public static final String AUTHORIZATION_STATUS = "authorizationStatus"; + + public static final String AUTHORIZATION_FAIL_REASON = "authorizationFailReason"; + + public static final String PAYER_IDENTIFIER = "payerIdentifier"; + + public static final String PARTY_LOOKUP_SUCCESSFUL_TRANSACTION_COUNT = "partyLookupSuccessfulTransactionAmount"; + + public static final String CURRENCY = "currency"; + + public static final String AUTHORIZATION_RESPONSE = "authorizationResponse"; + } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AuthorizationWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AuthorizationWorker.java new file mode 100644 index 00000000..ba651857 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AuthorizationWorker.java @@ -0,0 +1,77 @@ +package org.mifos.processor.bulk.zeebe.worker; + +import java.util.Map; +import org.mifos.processor.bulk.schema.AuthorizationRequest; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.AUTHORIZATION_SUCCESSFUL; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.APPROVED_AMOUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CLIENT_CORRELATION_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.AUTHORIZATION_ACCEPTED; + +@Component +public class AuthorizationWorker extends BaseWorker { + + @Value("${batch-authorization.callback-url}") + private String callbackURLPath; + + @Value("${mock-payment-schema.contactpoint}") + private String mockPaymentSchemaContactPoint; + + @Value("${mock-payment-schema.endpoints.authorization}") + private String authorizationEndpoint; + + private static final String X_CLIENT_CORRELATION_ID = "X-Client-Correlation-ID"; + + private static final String X_CALLBACK_URL = "X-CallbackURL"; + + @Override + public void setup() { + newWorker(Worker.AUTHORIZATION, (client, job) -> { + logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); + Map variables = job.getVariablesAsMap(); + + if (!workerConfig.isAuthorizationWorkerEnabled) { + variables.put(AUTHORIZATION_SUCCESSFUL, true); + client.newCompleteCommand(job.getKey()).variables(variables).send(); + return; + } + + String payerIdentifier = (String) variables.get("payerIdentifier"); + String totalBatchAmount = (String) variables.get("partyLookupSuccessfulTransactionAmount"); + String currency = (String) variables.get("currency"); + + String batchId = (String) variables.get(BATCH_ID); + String clientCorrelationId = Long.toString(job.getKey()); + + AuthorizationRequest requestPayload = new AuthorizationRequest(batchId, payerIdentifier, currency, totalBatchAmount); + HttpStatus httpStatus = invokeBatchAuthorizationApi(batchId, requestPayload, clientCorrelationId); + + variables.put(APPROVED_AMOUNT, totalBatchAmount); + variables.put(CLIENT_CORRELATION_ID, clientCorrelationId); + variables.put(AUTHORIZATION_ACCEPTED, httpStatus.is2xxSuccessful()); + client.newCompleteCommand(job.getKey()).variables(variables).send(); + }); + } + + private HttpStatus invokeBatchAuthorizationApi(String batchId, AuthorizationRequest requestPayload, String clientCorrelationId) { + RestTemplate restTemplate = new RestTemplate(); + HttpHeaders headers = new HttpHeaders(); + headers.add(X_CLIENT_CORRELATION_ID, clientCorrelationId); + headers.add(X_CALLBACK_URL, callbackURLPath); + + HttpEntity requestEntity = new HttpEntity<>(requestPayload, headers); + String endpoint = mockPaymentSchemaContactPoint + authorizationEndpoint + batchId; + + ResponseEntity responseEntity = restTemplate.exchange(endpoint, HttpMethod.POST, requestEntity, String.class); + return responseEntity.getStatusCode(); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java index 3e3e7daa..4e59a619 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java @@ -5,7 +5,8 @@ public enum Worker { PARTY_LOOKUP("partyLookup"), APPROVAL("approval"), ORDERING("ordering"), SPLITTING("splitting"), FORMATTING("formatting"), BATCH_STATUS("batchStatus"), SEND_CALLBACK("sendCallback"), MERGE_BACK("mergeSubBatch"), INIT_SUB_BATCH("initSubBatch"), ACCOUNT_LOOKUP("accountLookup"), - ACCOUNT_LOOKUP_CALLBACK("accountLookupCallback"), BATCH_AGGREGATE("batchAggregate"); + ACCOUNT_LOOKUP_CALLBACK("accountLookupCallback"), BATCH_AGGREGATE("batchAggregate"), + AUTHORIZATION("authorization"); private final String value; diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java index 56575c0e..29da2840 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/WorkerConfig.java @@ -33,4 +33,7 @@ public class WorkerConfig { @Value("${config.deduplication.enabled}") public boolean isTransactionDeduplicationEnabled; + @Value("${config.authorization.enabled}") + public boolean isAuthorizationWorkerEnabled; + } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 33e82a1b..b1b918be 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -38,6 +38,11 @@ operations-app: batch-transaction: "/api/v1/batch/transactions" batch-aggregate: "/api/v1/batch/" +mock-payment-schema: + contactpoint: "http://ph-ee-connector-mock-payment-schema:8080" + endpoints: + authorization: "/batches/" + channel: hostname: "https://ph-ee-connector-channel:8443" @@ -48,6 +53,7 @@ cloud: credentials: access-key: ${AWS_ACCESS_KEY:access_key_from_aws} secret-key: ${AWS_SECRET_KEY:secret_key_from_aws} + region: static: ap-south-1 stack: @@ -84,6 +90,8 @@ config: enable: true partylookup: enable: true + authorization: + enabled: true approval: enable: true ordering: @@ -182,3 +190,5 @@ payment-mode: type: "BULK" endpoint: "bulk_connector_{MODE}-{dfspid}" debulkingDfspid: "lion" +batch-authorization: + callback-url: "https://ph-ee-connector-bulk:80/authorization/callback" From 9bf984fcf16105081ff53936f72405814ef8ba71 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Mon, 30 Oct 2023 12:31:06 +0530 Subject: [PATCH 098/156] GOV-325-fix: Fixed the authorisation API call (#123) * Added logger for testing auth * Fixed the mock-payemnt-schema api call * test * test 1 * Updated the log statements --- .../processor/bulk/api/CallbackController.java | 12 +++++++++++- .../bulk/zeebe/worker/AuthorizationWorker.java | 17 ++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/api/CallbackController.java b/src/main/java/org/mifos/processor/bulk/api/CallbackController.java index 76a2d82c..8d1a5be6 100644 --- a/src/main/java/org/mifos/processor/bulk/api/CallbackController.java +++ b/src/main/java/org/mifos/processor/bulk/api/CallbackController.java @@ -1,5 +1,7 @@ package org.mifos.processor.bulk.api; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import io.camunda.zeebe.client.ZeebeClient; import java.time.Duration; import java.util.HashMap; @@ -21,13 +23,18 @@ public class CallbackController { @Autowired private ZeebeClient zeebeClient; + @Autowired + ObjectMapper objectMapper; + protected Logger logger = LoggerFactory.getLogger(this.getClass()); private static final String EXPECTED_AUTH_STATUS = "Y"; @PostMapping("/authorization/callback") - public ResponseEntity handleAuthorizationCallback(@RequestBody AuthorizationResponse authResponse) { + public ResponseEntity handleAuthorizationCallback(@RequestBody AuthorizationResponse authResponse) throws JsonProcessingException { + logger.info("Callback received"); + logger.debug("Auth response: {}", objectMapper.writeValueAsString(authResponse)); Map variables = new HashMap<>(); boolean isAuthorizationSuccessful = EXPECTED_AUTH_STATUS.equals(authResponse.getStatus()); @@ -40,12 +47,15 @@ public ResponseEntity handleAuthorizationCallback(@RequestBody Authoriza variables.put(APPROVED_AMOUNT, 0); } + logger.info("Is auth successful: {}", isAuthorizationSuccessful); + if (zeebeClient != null) { zeebeClient.newPublishMessageCommand() .messageName(AUTHORIZATION_RESPONSE) .correlationKey(authResponse.getClientCorrelationId()) .timeToLive(Duration.ofMillis(500000)) .variables(variables).send(); + logger.debug("Published zeebe message event {}", AUTHORIZATION_RESPONSE); } return ResponseEntity.ok().build(); } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AuthorizationWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AuthorizationWorker.java index ba651857..42e919b3 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AuthorizationWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AuthorizationWorker.java @@ -1,7 +1,12 @@ package org.mifos.processor.bulk.zeebe.worker; +import java.util.HashMap; import java.util.Map; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import org.mifos.processor.bulk.schema.AuthorizationRequest; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; @@ -33,6 +38,9 @@ public class AuthorizationWorker extends BaseWorker { private static final String X_CALLBACK_URL = "X-CallbackURL"; + @Autowired + ObjectMapper objectMapper; + @Override public void setup() { newWorker(Worker.AUTHORIZATION, (client, job) -> { @@ -45,6 +53,8 @@ public void setup() { return; } + logger.debug("Variables: {}", variables); + String payerIdentifier = (String) variables.get("payerIdentifier"); String totalBatchAmount = (String) variables.get("partyLookupSuccessfulTransactionAmount"); String currency = (String) variables.get("currency"); @@ -62,7 +72,8 @@ public void setup() { }); } - private HttpStatus invokeBatchAuthorizationApi(String batchId, AuthorizationRequest requestPayload, String clientCorrelationId) { + private HttpStatus invokeBatchAuthorizationApi(String batchId, AuthorizationRequest requestPayload, String clientCorrelationId) throws JsonProcessingException { + logger.info("Calling auth API"); RestTemplate restTemplate = new RestTemplate(); HttpHeaders headers = new HttpHeaders(); headers.add(X_CLIENT_CORRELATION_ID, clientCorrelationId); @@ -70,7 +81,11 @@ private HttpStatus invokeBatchAuthorizationApi(String batchId, AuthorizationRequ HttpEntity requestEntity = new HttpEntity<>(requestPayload, headers); String endpoint = mockPaymentSchemaContactPoint + authorizationEndpoint + batchId; + endpoint = endpoint + "?command=authorize"; + logger.debug("Auth API request headers: {}", headers); + logger.debug("Endpoint: {}", endpoint); + logger.debug("Body: {}", objectMapper.writeValueAsString(requestPayload)); ResponseEntity responseEntity = restTemplate.exchange(endpoint, HttpMethod.POST, requestEntity, String.class); return responseEntity.getStatusCode(); } From 647d9d946059b3739f7385f501cec9e52d126b4d Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Tue, 31 Oct 2023 15:11:52 +0530 Subject: [PATCH 099/156] PHEE-457 : Fixed the payload while sending auth request to mockpaymentschema (#125) * Fixed the payload while sending auth request to mockpaymentschema * Test * Test * Updated connector-comon * Reverted the callback-url config * Loggers level updated * Updated PR * Fixed the connector-common 1.7.0 version --- build.gradle | 2 +- .../processor/bulk/zeebe/worker/AuthorizationWorker.java | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 59f43ed6..04653425 100644 --- a/build.gradle +++ b/build.gradle @@ -108,7 +108,7 @@ dependencies { // miscellaneous dependency implementation 'com.google.code.gson:gson:2.8.9' implementation 'org.json:json:20210307' - implementation 'org.mifos:ph-ee-connector-common:1.6.0' + implementation 'org.mifos:ph-ee-connector-common:1.7.0-SNAPSHOT' implementation 'org.apache.camel.springboot:camel-spring-boot-starter:3.4.0' implementation 'org.apache.camel:camel-undertow:3.4.0' implementation 'org.springframework.boot:spring-boot-starter:2.5.2' diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AuthorizationWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AuthorizationWorker.java index 42e919b3..fd08ba5b 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AuthorizationWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AuthorizationWorker.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import io.micrometer.core.instrument.util.StringUtils; import org.mifos.processor.bulk.schema.AuthorizationRequest; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -65,6 +66,8 @@ public void setup() { AuthorizationRequest requestPayload = new AuthorizationRequest(batchId, payerIdentifier, currency, totalBatchAmount); HttpStatus httpStatus = invokeBatchAuthorizationApi(batchId, requestPayload, clientCorrelationId); + logger.info("Httpstatus: {}", httpStatus); + variables.put(APPROVED_AMOUNT, totalBatchAmount); variables.put(CLIENT_CORRELATION_ID, clientCorrelationId); variables.put(AUTHORIZATION_ACCEPTED, httpStatus.is2xxSuccessful()); @@ -74,6 +77,9 @@ public void setup() { private HttpStatus invokeBatchAuthorizationApi(String batchId, AuthorizationRequest requestPayload, String clientCorrelationId) throws JsonProcessingException { logger.info("Calling auth API"); + if (StringUtils.isBlank(requestPayload.getAmount())) { + requestPayload.setAmount("0"); + } RestTemplate restTemplate = new RestTemplate(); HttpHeaders headers = new HttpHeaders(); headers.add(X_CLIENT_CORRELATION_ID, clientCorrelationId); @@ -84,7 +90,7 @@ private HttpStatus invokeBatchAuthorizationApi(String batchId, AuthorizationRequ endpoint = endpoint + "?command=authorize"; logger.debug("Auth API request headers: {}", headers); - logger.debug("Endpoint: {}", endpoint); + logger.info("MockPaymentSchema endpoint: {}", endpoint); logger.debug("Body: {}", objectMapper.writeValueAsString(requestPayload)); ResponseEntity responseEntity = restTemplate.exchange(endpoint, HttpMethod.POST, requestEntity, String.class); return responseEntity.getStatusCode(); From 96e3d7dbebb5834ee603f099933d33ea3b4acd87 Mon Sep 17 00:00:00 2001 From: Apurb Rajdhan <103931815+apurbraj@users.noreply.github.com> Date: Wed, 1 Nov 2023 13:32:53 +0530 Subject: [PATCH 100/156] GOV-517 (#114) --- .../AccountLookupCallbackProcessor.java | 60 ------------------ .../bulk/camel/routes/AccountLookupRoute.java | 61 ++++++++++--------- .../bulk/camel/routes/InitSubBatchRoute.java | 18 +----- .../bulk/schema/AccountLookupResponseDTO.java | 1 + .../worker/AccountLookupCallbackWorker.java | 5 +- .../zeebe/worker/AccountLookupWorker.java | 19 +++++- src/main/resources/application.yaml | 10 +-- 7 files changed, 62 insertions(+), 112 deletions(-) delete mode 100644 src/main/java/org/mifos/processor/bulk/camel/processor/AccountLookupCallbackProcessor.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/processor/AccountLookupCallbackProcessor.java b/src/main/java/org/mifos/processor/bulk/camel/processor/AccountLookupCallbackProcessor.java deleted file mode 100644 index 3065e73a..00000000 --- a/src/main/java/org/mifos/processor/bulk/camel/processor/AccountLookupCallbackProcessor.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.mifos.processor.bulk.camel.processor; - -import static org.mifos.processor.bulk.camel.config.CamelProperties.CACHED_TRANSACTION_ID; -import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_PARTY_ID; -import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_PARTY_ID_TYPE; -import static org.mifos.processor.bulk.zeebe.ZeebeMessages.ACCOUNT_LOOKUP; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ACCOUNT_LOOKUP_FAILED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_FSP_ID; - -import com.fasterxml.jackson.databind.ObjectMapper; -import io.camunda.zeebe.client.ZeebeClient; -import java.io.IOException; -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; -import org.apache.camel.Exchange; -import org.apache.camel.Processor; -import org.mifos.processor.bulk.schema.AccountLookupResponseDTO; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -@Component -public class AccountLookupCallbackProcessor implements Processor { - - private final Logger logger = LoggerFactory.getLogger(this.getClass()); - - @Autowired(required = false) - private ZeebeClient zeebeClient; - @Autowired - private ObjectMapper objectMapper; - - @Override - public void process(Exchange exchange) throws Exception { - Map variables = new HashMap<>(); - - String error = null; - String response = exchange.getIn().getBody(String.class); - AccountLookupResponseDTO accountLookupResponseDTO = null; - try { - accountLookupResponseDTO = objectMapper.readValue(response, AccountLookupResponseDTO.class); - variables.put(ACCOUNT_LOOKUP_FAILED, false); - variables.put(PAYEE_PARTY_ID, accountLookupResponseDTO.getPaymentModalityList().get(0).getFinancialAddress()); - variables.put(PAYEE_PARTY_ID_TYPE, accountLookupResponseDTO.getPaymentModalityList().get(0).getPaymentModality()); - variables.put(PARTY_LOOKUP_FSP_ID, accountLookupResponseDTO.getPaymentModalityList().get(0).getBankingInstitutionCode()); - exchange.setProperty(CACHED_TRANSACTION_ID, accountLookupResponseDTO.getRequestId()); - } catch (IOException e) { - variables.put(ACCOUNT_LOOKUP_FAILED, true); - error = objectMapper.readValue(response, String.class); - } - - if (zeebeClient != null) { - - zeebeClient.newPublishMessageCommand().messageName(ACCOUNT_LOOKUP) - .correlationKey(exchange.getProperty(CACHED_TRANSACTION_ID, String.class)).timeToLive(Duration.ofMillis(50000)) - .variables(variables).send(); - } - } -} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java index 7a3b83fd..bd8a2862 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java @@ -1,39 +1,44 @@ package org.mifos.processor.bulk.camel.routes; -import static org.mifos.processor.bulk.camel.config.CamelProperties.ENDPOINT; -import static org.mifos.processor.bulk.camel.config.CamelProperties.HOST; -import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_IDENTITY; -import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYMENT_MODALITY; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; - import org.apache.camel.Exchange; -import org.apache.camel.LoggingLevel; -import org.mifos.processor.bulk.camel.processor.AccountLookupCallbackProcessor; -import org.springframework.beans.factory.annotation.Autowired; +import org.apache.camel.Processor; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -@Component -public class AccountLookupRoute extends BaseRouteBuilder { +import javax.net.ssl.HttpsURLConnection; - @Autowired - AccountLookupCallbackProcessor accountLookupCallbackProcessor; +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; +@Component +public class AccountLookupRoute extends BaseRouteBuilder { + @Value("${identity_account_mapper.account_lookup}") + private String accountLookupEndpoint; + @Value("${identity_account_mapper.hostname}") + private String identityURL; @Override public void configure() throws Exception { - - from("rest:POST:/accountLookup/Callback").log(LoggingLevel.DEBUG, "######## -> ACCOUNT LOOKUP CALLBACK") - .process(accountLookupCallbackProcessor).setBody(constant("Received")).setHeader(Exchange.HTTP_RESPONSE_CODE, constant(200)) - .end(); - - from("direct:send-account-lookup").id("account-lookup").process(exchange -> { - String callbackUrl = exchange.getProperty(CALLBACK, String.class); - exchange.getIn().setHeader(CALLBACK, callbackUrl); - }).setHeader(Exchange.HTTP_METHOD, constant("GET")) - .setHeader(Exchange.HTTP_QUERY, - simple(new StringBuilder().append(PAYEE_IDENTITY).append("=${exchangeProperty.").append(PAYEE_IDENTITY).append("}&") - .append(PAYMENT_MODALITY).append("=${exchangeProperty.").append(PAYMENT_MODALITY).append("}&") - .append("requestId=${exchangeProperty.requestId}").toString())) - .setProperty(HOST, simple("{{identity-account-mapper.hostname}}")) - .setProperty(ENDPOINT, simple("identityAccountMapper/accountLookup/")).to("direct:external-api-calling"); + Processor disableSslProcessor = new Processor() { + @Override + public void process(Exchange exchange) throws Exception { + // Disable SSL certificate validation + HttpsURLConnection.setDefaultHostnameVerifier((hostname, sslSession) -> true); + } + }; + from("direct:send-account-lookup") + .id("account-lookup") + .process(exchange -> { + String callbackUrl = exchange.getProperty(CALLBACK, String.class); + String registeringInstitutionId = exchange.getProperty(HEADER_REGISTERING_INSTITUTE_ID, String.class); + exchange.getIn().setHeader(CALLBACK, callbackUrl); + exchange.getIn().setHeader(HEADER_REGISTERING_INSTITUTE_ID, registeringInstitutionId); + }) + .setHeader(Exchange.HTTP_METHOD, constant("GET")) + .toD(identityURL + accountLookupEndpoint + "?" + + PAYEE_IDENTITY + "=${exchangeProperty.payeeIdentity}&" + + PAYMENT_MODALITY + "=${exchangeProperty.paymentModality}&" + + "requestId=${exchangeProperty.requestId}") + .log("API Response: ${body}") + .process(disableSslProcessor); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index c8bf3585..df306496 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -1,20 +1,7 @@ package org.mifos.processor.bulk.camel.routes; -import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_ID_HEADER; -import static org.mifos.processor.bulk.camel.config.CamelProperties.EXTERNAL_ENDPOINT; -import static org.mifos.processor.bulk.camel.config.CamelProperties.EXTERNAL_ENDPOINT_FAILED; -import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_CLIENT_CORRELATION_ID; -import static org.mifos.processor.bulk.camel.config.CamelProperties.IS_PAYMENT_MODE_VALID; -import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; -import static org.mifos.processor.bulk.camel.config.CamelProperties.OVERRIDE_HEADER; -import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYMENT_MODE_TYPE; -import static org.mifos.processor.bulk.camel.config.CamelProperties.RESULT_TRANSACTION_LIST; -import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; -import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; -import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; -import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_ELEMENT; -import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_LENGTH; -import static org.mifos.processor.bulk.camel.config.CamelProperties.ZEEBE_VARIABLE; +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETED_AMOUNT; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DEBULKINGDFSPID; @@ -197,6 +184,7 @@ public void configure() throws Exception { .setHeader(Exchange.CONTENT_TYPE, constant("application/json")) .setHeader(BATCH_ID_HEADER, simple("${exchangeProperty." + BATCH_ID + "}")) .setHeader(HEADER_CLIENT_CORRELATION_ID, simple("${exchangeProperty." + REQUEST_ID + "}")) + .setHeader(HEADER_REGISTERING_INSTITUTE_ID, simple("${exchangeProperty." + HEADER_REGISTERING_INSTITUTE_ID + "}")) .process(exchange -> { log.debug("Variables: {}", exchange.getProperties()); log.debug("Emergency: {}", exchange.getIn().getHeaders()); diff --git a/src/main/java/org/mifos/processor/bulk/schema/AccountLookupResponseDTO.java b/src/main/java/org/mifos/processor/bulk/schema/AccountLookupResponseDTO.java index edadd569..e511edf1 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/AccountLookupResponseDTO.java +++ b/src/main/java/org/mifos/processor/bulk/schema/AccountLookupResponseDTO.java @@ -16,5 +16,6 @@ public class AccountLookupResponseDTO implements Serializable { private String requestId; private String payeeIdentity; private List paymentModalityList; + private Boolean isValidated; } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java index db31b70a..25a96055 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java @@ -1,8 +1,7 @@ package org.mifos.processor.bulk.zeebe.worker; import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_PARTY_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CHANNEL_REQUEST; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORIGIN_CHANNEL_REQUEST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; import static org.mifos.processor.bulk.zeebe.worker.Worker.ACCOUNT_LOOKUP_CALLBACK; import com.fasterxml.jackson.databind.ObjectMapper; @@ -35,7 +34,9 @@ public void setup() { TransactionChannelRequestDTO transactionChannelRequestDTO = objectMapper .readValue((String) existingVariables.get(CHANNEL_REQUEST), TransactionChannelRequestDTO.class); String payeeId = existingVariables.get(PAYEE_PARTY_ID).toString(); + String payeeFspId = existingVariables.get(PARTY_LOOKUP_FSP_ID).toString(); PartyIdInfo partyIdInfo = new PartyIdInfo(transactionChannelRequestDTO.getPayee().getPartyIdInfo().getPartyIdType(), payeeId); + partyIdInfo.setFspId(payeeFspId); Party payee = new Party(partyIdInfo); transactionChannelRequestDTO.setPayee(payee); existingVariables.put(CHANNEL_REQUEST, objectMapper.writeValueAsString(transactionChannelRequestDTO)); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java index 51e84b6e..72962261 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java @@ -4,6 +4,7 @@ import static org.mifos.processor.bulk.camel.config.CamelProperties.HOST; import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_IDENTITY; import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYMENT_MODALITY; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ACCOUNT_LOOKUP_RETRY_COUNT; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CHANNEL_REQUEST; @@ -17,15 +18,19 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.camunda.zeebe.client.ZeebeClient; import java.util.Map; + import org.apache.camel.CamelContext; import org.apache.camel.Exchange; import org.apache.camel.ProducerTemplate; import org.apache.camel.support.DefaultExchange; import org.mifos.connector.common.channel.dto.TransactionChannelRequestDTO; import org.mifos.connector.common.mojaloop.dto.PartyIdInfo; +import org.mifos.processor.bulk.schema.AuthorizationRequest; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.*; import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; @Component public class AccountLookupWorker extends BaseWorker { @@ -39,10 +44,14 @@ public class AccountLookupWorker extends BaseWorker { private CamelContext camelContext; @Autowired private ObjectMapper objectMapper; - @Value("${identity-account-mapper.hostname}") + @Value("${identity_account_mapper.hostname}") private String identityMapperURL; - @Value("${bulk-processor.hostname}") + @Value("${identity_account_mapper.account_lookup_callback}") + private String accountLookupCallback; + @Value("${bulk_processor.hostname}") private String bulkURL; + @Value("${identity_account_mapper.account_lookup}") + private String accountLookupEndpoint; @Override public void setup() { @@ -50,6 +59,7 @@ public void setup() { zeebeClient.newWorker().jobType(ACCOUNT_LOOKUP.getValue()).handler((client, job) -> { logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); Map existingVariables = job.getVariablesAsMap(); + logger.info(existingVariables.toString()); existingVariables.put(ACCOUNT_LOOKUP_RETRY_COUNT, 1); existingVariables.put(CACHED_TRANSACTION_ID, job.getKey()); @@ -68,16 +78,19 @@ public void setup() { exchange.setProperty(HOST, identityMapperURL); exchange.setProperty(PAYEE_IDENTITY, payeeIdentity); exchange.setProperty(PAYMENT_MODALITY, paymentModality); - exchange.setProperty(CALLBACK, bulkURL + "/accountLookup/Callback"); + exchange.setProperty(CALLBACK, identityMapperURL + accountLookupCallback); exchange.setProperty(TRANSACTION_ID, existingVariables.get(TRANSACTION_ID)); exchange.setProperty("requestId", job.getKey()); exchange.setProperty(CHANNEL_REQUEST, channelRequest); exchange.setProperty(ORIGIN_DATE, existingVariables.get(ORIGIN_DATE)); exchange.setProperty(TENANT_ID, tenantId); + exchange.setProperty(HEADER_REGISTERING_INSTITUTE_ID, existingVariables.get(HEADER_REGISTERING_INSTITUTE_ID)); producerTemplate.send("direct:send-account-lookup", exchange); + client.newCompleteCommand(job.getKey()).variables(existingVariables).send(); }).name(String.valueOf(ACCOUNT_LOOKUP)).open(); } + } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index b1b918be..f4fc2a29 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -146,11 +146,13 @@ security: response: enable: true -identity-account-mapper: - hostname : "https://identity.sandbox.mifos.io" +identity_account_mapper: + hostname : "http://ph-ee-identity-account-mapper:80" + account_lookup: /beneficiary + account_lookup_callback: /accountLookupCallback -bulk-processor: - hostname : "https://bulk-connector.sandbox.mifos.io" +bulk_processor: + hostname : "http://ph-ee-connector-bulk:8080" csv: columnNames: "id,request_id,payment_mode,payer_identifier_type,payer_identifier,payee_identifier_type,payee_identifier,amount,currency,note,account_number,program_shortcode,cycle" From 398050ef759336c1d7a9283966ad9f0d37dcd763 Mon Sep 17 00:00:00 2001 From: Dhruv Sonagara <78945411+dhruvsonagara@users.noreply.github.com> Date: Fri, 3 Nov 2023 17:39:51 +0530 Subject: [PATCH 101/156] PHEE-307 Resolve checkstyle errors manually and update gradle command in CI (#107) * PHEE-307 Resolve checkstyle errors manually and update gradle command in CI * updated the gradle command for checkstyle in CI * Rebased and resolved conflicts * Resolved the checkstyle error manually and updated the api package name * Applied the spotless check --------- Co-authored-by: Dhruv Sonagara --- .circleci/config.yml | 1 + build.gradle | 32 +------ .../processor/BulkProcessorApplication.java | 4 +- .../bulk/ConfigurationValidator.java | 2 +- .../processor/bulk/api/ApiOriginFilter.java | 10 +-- .../bulk/api/CallbackController.java | 22 +++-- .../definition/BatchTransactions.java | 34 +++----- .../{API => api}/definition/BulkTransfer.java | 10 +-- .../{API => api}/definition/Simulate.java | 6 +- .../BatchTransactionsController.java | 83 +++++++------------ .../BulkTransferController.java | 13 +-- .../implementation/SimulateApiController.java | 2 +- .../bulk/camel/config/CamelProperties.java | 2 +- .../HttpClientConfigurerTrustAllCACerts.java | 2 +- .../processor/DTOJsonConversionException.java | 4 +- .../bulk/camel/processor/GsmaApiPayload.java | 4 +- .../camel/processor/MojaloopApiPayload.java | 2 +- .../bulk/camel/routes/AccountLookupRoute.java | 38 ++++----- .../camel/routes/BatchAggregateRoute.java | 21 ++--- .../camel/routes/FileProcessingRoute.java | 2 +- .../bulk/camel/routes/FileRoute.java | 1 - .../bulk/camel/routes/HealthRoute.java | 19 +++++ .../bulk/camel/routes/InitSubBatchRoute.java | 32 ++++--- .../bulk/camel/routes/OrderingRoute.java | 9 +- .../camel/routes/ProcessorStartRoute.java | 60 ++++++++------ .../processor/bulk/camel/routes/RouteId.java | 9 +- .../bulk/camel/routes/SendCallbackRoute.java | 12 ++- .../bulk/config/BudgetAccountConfig.java | 9 +- .../mifos/processor/bulk/config/Program.java | 5 +- .../config/RegisteringInstitutionConfig.java | 9 +- .../bulk/file/AwsFileTransferImpl.java | 4 +- .../bulk/file/AzureFileTransferImpl.java | 9 +- .../bulk/file/FileStorageService.java | 5 +- .../bulk/file/FileStorageServiceImpl.java | 9 +- .../bulk/format/RestRequestConvertor.java | 7 +- .../processor/bulk/format/StandardValue.java | 4 +- .../bulk/format/helper/BaseMapper.java | 6 +- .../processor/bulk/format/helper/Mapper.java | 4 +- .../mifos/processor/bulk/schema/BatchDTO.java | 9 +- .../bulk/schema/BatchRequestDTO.java | 11 ++- .../mifos/processor/bulk/schema/Party.java | 3 +- .../processor/bulk/schema/Transaction.java | 4 +- .../processor/bulk/utility/CsvWriter.java | 5 +- .../mifos/processor/bulk/utility/Headers.java | 7 +- .../bulk/utility/SpringWrapperUtil.java | 9 +- .../mifos/processor/bulk/utility/Utils.java | 19 +++-- .../processor/bulk/zeebe/ZeebeMessages.java | 2 +- .../processor/bulk/zeebe/ZeebeVariables.java | 10 +-- .../processor/bulk/zeebe/ZeebeWorkers.java | 3 +- .../worker/AccountLookupCallbackWorker.java | 4 +- .../zeebe/worker/AccountLookupWorker.java | 7 +- .../bulk/zeebe/worker/AggregateWorker.java | 2 +- .../zeebe/worker/AuthorizationWorker.java | 17 ++-- .../bulk/zeebe/worker/BatchStatusWorker.java | 23 ++--- .../processor/bulk/zeebe/worker/Worker.java | 11 ++- .../BulkProcessorApplicationTests.java | 6 +- .../cucumber/stepdef/InitRouteStepDef.java | 26 +++--- 57 files changed, 326 insertions(+), 359 deletions(-) rename src/main/java/org/mifos/processor/bulk/{API => api}/definition/BatchTransactions.java (54%) rename src/main/java/org/mifos/processor/bulk/{API => api}/definition/BulkTransfer.java (100%) rename src/main/java/org/mifos/processor/bulk/{API => api}/definition/Simulate.java (83%) rename src/main/java/org/mifos/processor/bulk/{API => api}/implementation/BatchTransactionsController.java (79%) rename src/main/java/org/mifos/processor/bulk/{API => api}/implementation/BulkTransferController.java (99%) rename src/main/java/org/mifos/processor/bulk/{API => api}/implementation/SimulateApiController.java (100%) create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/HealthRoute.java diff --git a/.circleci/config.yml b/.circleci/config.yml index 3e699b75..ade07cb9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,6 +57,7 @@ jobs: - run: name: Build Docker image command: | + ./gradlew checkstyleMain ./gradlew bootJar docker build -t openmf/ph-ee-bulk-processor:latest . diff --git a/build.gradle b/build.gradle index 04653425..23ed3b42 100644 --- a/build.gradle +++ b/build.gradle @@ -9,8 +9,6 @@ plugins { id 'checkstyle' id 'org.springframework.boot' version '2.6.2' id 'com.diffplug.spotless' version '6.19.0' - id 'com.github.hierynomus.license' version '0.16.1' - id 'com.github.jk1.dependency-license-report' version '2.1' } repositories { @@ -44,32 +42,6 @@ spotless { } lineEndings 'UNIX' } -// Configuration for Gradle license plug-in -// https://github.com/hierynomus/license-gradle-plugin -license { - header rootProject.file("$rootDir/LICENSE.md") - excludes([ - "**/gradlew*", - "**/git.properties", - "**/*.html", - "**/templates/**", - "**/features/**", - "**/results/**", - "**/package-info.java", - "**/keystore.jks", - "**/legacy-docs/**", - "**/banner.txt", - "**/build.gradle.mustache", - "**/pom.mustache", - "**/avro/**/*.java", - "**/org/apache/fineract/client/**/*.java" - ]) - strictCheck true -} - -licenseReport { - outputDir = "$projectDir/licenses" -} ext { camelVersion = '3.4.0' @@ -145,9 +117,7 @@ configure(this) { apply plugin: 'eclipse' apply plugin: 'checkstyle' apply plugin: 'com.diffplug.spotless' - apply plugin: 'com.github.hierynomus.license' - apply plugin: 'project-report' - apply plugin: 'com.github.jk1.dependency-license-report' + configurations { implementation.setCanBeResolved(true) api.setCanBeResolved(true) diff --git a/src/main/java/org/mifos/processor/BulkProcessorApplication.java b/src/main/java/org/mifos/processor/BulkProcessorApplication.java index a73fd9cd..79f1e958 100644 --- a/src/main/java/org/mifos/processor/BulkProcessorApplication.java +++ b/src/main/java/org/mifos/processor/BulkProcessorApplication.java @@ -11,9 +11,7 @@ import org.mifos.processor.bulk.api.ApiOriginFilter; import org.mifos.processor.bulk.camel.config.HttpClientConfigurerTrustAllCACerts; import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.context.annotation.Bean; @@ -56,7 +54,7 @@ public FilterRegistrationBean apiOriginFilter() { registration.setFilter(new ApiOriginFilter()); registration.addUrlPatterns("/**"); registration.setName("apiOriginFilter"); - registration.setOrder(Integer.MIN_VALUE+1); + registration.setOrder(Integer.MIN_VALUE + 1); return registration; } diff --git a/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java b/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java index 826f6133..516d3227 100644 --- a/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java +++ b/src/main/java/org/mifos/processor/bulk/ConfigurationValidator.java @@ -62,7 +62,7 @@ private void validateFormattingStandard() { logger.info("Configured formatting standard as >> {}", standardEnum.name()); return; } catch (Exception e) { - e.printStackTrace(); + logger.debug(e.getMessage()); } List possibleStandards = new ArrayList<>(); for (Field f : Standard.class.getFields()) { diff --git a/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java b/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java index 8db48fa3..4137a6cf 100644 --- a/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java +++ b/src/main/java/org/mifos/processor/bulk/api/ApiOriginFilter.java @@ -1,20 +1,17 @@ package org.mifos.processor.bulk.api; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; + import java.io.IOException; -import javax.servlet.Filter; import javax.servlet.FilterChain; -import javax.servlet.FilterConfig; import javax.servlet.ServletException; import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; import org.springframework.web.filter.GenericFilterBean; -import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; - public class ApiOriginFilter extends GenericFilterBean { private Logger logger = LoggerFactory.getLogger(this.getClass()); @@ -22,8 +19,7 @@ public class ApiOriginFilter extends GenericFilterBean { @Override public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { HttpServletRequest req = (HttpServletRequest) request; - String tenant = req.getHeader("" + - HEADER_PLATFORM_TENANT_ID); + String tenant = req.getHeader("" + HEADER_PLATFORM_TENANT_ID); logger.debug("Tenant Name is : {}", tenant); logger.info("Client IP Address: {}", req.getRemoteHost()); } diff --git a/src/main/java/org/mifos/processor/bulk/api/CallbackController.java b/src/main/java/org/mifos/processor/bulk/api/CallbackController.java index 8d1a5be6..8dab4d5b 100644 --- a/src/main/java/org/mifos/processor/bulk/api/CallbackController.java +++ b/src/main/java/org/mifos/processor/bulk/api/CallbackController.java @@ -1,5 +1,12 @@ package org.mifos.processor.bulk.api; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.APPROVED_AMOUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.AUTHORIZATION_FAIL_REASON; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.AUTHORIZATION_RESPONSE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.AUTHORIZATION_STATUS; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.AUTHORIZATION_SUCCESSFUL; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CLIENT_CORRELATION_ID; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.camunda.zeebe.client.ZeebeClient; @@ -15,8 +22,6 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; - @RestController public class CallbackController { @@ -30,9 +35,9 @@ public class CallbackController { private static final String EXPECTED_AUTH_STATUS = "Y"; - @PostMapping("/authorization/callback") - public ResponseEntity handleAuthorizationCallback(@RequestBody AuthorizationResponse authResponse) throws JsonProcessingException { + public ResponseEntity handleAuthorizationCallback(@RequestBody AuthorizationResponse authResponse) + throws JsonProcessingException { logger.info("Callback received"); logger.debug("Auth response: {}", objectMapper.writeValueAsString(authResponse)); Map variables = new HashMap<>(); @@ -50,12 +55,11 @@ public ResponseEntity handleAuthorizationCallback(@RequestBody Authoriza logger.info("Is auth successful: {}", isAuthorizationSuccessful); if (zeebeClient != null) { - zeebeClient.newPublishMessageCommand() - .messageName(AUTHORIZATION_RESPONSE) - .correlationKey(authResponse.getClientCorrelationId()) - .timeToLive(Duration.ofMillis(500000)) - .variables(variables).send(); + zeebeClient.newPublishMessageCommand().messageName(AUTHORIZATION_RESPONSE).correlationKey(authResponse.getClientCorrelationId()) + .timeToLive(Duration.ofMillis(500000)).variables(variables).send(); logger.debug("Published zeebe message event {}", AUTHORIZATION_RESPONSE); + zeebeClient.newPublishMessageCommand().messageName(AUTHORIZATION_RESPONSE).correlationKey(authResponse.getClientCorrelationId()) + .timeToLive(Duration.ofMillis(500000)).variables(variables).send(); } return ResponseEntity.ok().build(); } diff --git a/src/main/java/org/mifos/processor/bulk/API/definition/BatchTransactions.java b/src/main/java/org/mifos/processor/bulk/api/definition/BatchTransactions.java similarity index 54% rename from src/main/java/org/mifos/processor/bulk/API/definition/BatchTransactions.java rename to src/main/java/org/mifos/processor/bulk/api/definition/BatchTransactions.java index e287bfcc..119ed31e 100644 --- a/src/main/java/org/mifos/processor/bulk/API/definition/BatchTransactions.java +++ b/src/main/java/org/mifos/processor/bulk/api/definition/BatchTransactions.java @@ -1,34 +1,26 @@ package org.mifos.processor.bulk.api.definition; -import com.amazonaws.services.dynamodbv2.xspec.S; -import com.fasterxml.jackson.core.JsonProcessingException; -import org.apache.camel.util.json.JsonObject; -import org.json.JSONObject; -import org.springframework.web.bind.annotation.*; -import org.springframework.web.multipart.MultipartFile; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_CLIENT_CORRELATION_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PROGRAM_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_TYPE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; -import javax.mail.Multipart; +import java.io.IOException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; - -import java.io.IOException; - -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; -import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_CLIENT_CORRELATION_ID; -import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestHeader; public interface BatchTransactions { @PostMapping(value = "/batchtransactions", produces = "application/json") - String batchTransactions( - HttpServletRequest httpServletRequest, - HttpServletResponse httpServletResponse, + String batchTransactions(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, @RequestHeader(value = HEADER_CLIENT_CORRELATION_ID) String requestId, - @RequestHeader(value = FILE_NAME, required = false) String fileName, - @RequestHeader(value = PURPOSE) String purpose, - @RequestHeader(value = HEADER_TYPE) String type, - @RequestHeader(value = HEADER_PLATFORM_TENANT_ID) String tenant, + @RequestHeader(value = FILE_NAME, required = false) String fileName, @RequestHeader(value = PURPOSE) String purpose, + @RequestHeader(value = HEADER_TYPE) String type, @RequestHeader(value = HEADER_PLATFORM_TENANT_ID) String tenant, @RequestHeader(value = HEADER_REGISTERING_INSTITUTE_ID, required = false) String registeringInstitutionId, @RequestHeader(value = HEADER_PROGRAM_ID, required = false) String programId) throws IOException; diff --git a/src/main/java/org/mifos/processor/bulk/API/definition/BulkTransfer.java b/src/main/java/org/mifos/processor/bulk/api/definition/BulkTransfer.java similarity index 100% rename from src/main/java/org/mifos/processor/bulk/API/definition/BulkTransfer.java rename to src/main/java/org/mifos/processor/bulk/api/definition/BulkTransfer.java index 5e0a3325..d081c28d 100644 --- a/src/main/java/org/mifos/processor/bulk/API/definition/BulkTransfer.java +++ b/src/main/java/org/mifos/processor/bulk/api/definition/BulkTransfer.java @@ -1,18 +1,18 @@ package org.mifos.processor.bulk.api.definition; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; + +import java.io.IOException; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.multipart.MultipartFile; -import java.io.IOException; - -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; - public interface BulkTransfer { @Deprecated + @PostMapping(value = "/bulk/transfer/{requestId}/{fileName}", produces = "application/json") String bulkTransfer(@RequestHeader(value = "X-CorrelationID", required = false) String requestId, @RequestParam("data") MultipartFile file, @RequestHeader(value = FILE_NAME, required = false) String fileName, diff --git a/src/main/java/org/mifos/processor/bulk/API/definition/Simulate.java b/src/main/java/org/mifos/processor/bulk/api/definition/Simulate.java similarity index 83% rename from src/main/java/org/mifos/processor/bulk/API/definition/Simulate.java rename to src/main/java/org/mifos/processor/bulk/api/definition/Simulate.java index 81518166..91406d60 100644 --- a/src/main/java/org/mifos/processor/bulk/API/definition/Simulate.java +++ b/src/main/java/org/mifos/processor/bulk/api/definition/Simulate.java @@ -1,13 +1,13 @@ package org.mifos.processor.bulk.api.definition; -import org.springframework.web.bind.annotation.PostMapping; -import javax.servlet.http.HttpServletResponse; import java.io.IOException; +import javax.servlet.http.HttpServletResponse; +import org.springframework.web.bind.annotation.PostMapping; // from("rest:post:/simulate").log("Reached Simulation"); public interface Simulate { - @PostMapping(value = "/simulate", produces="application/json") + @PostMapping(value = "/simulate", produces = "application/json") void simulate(HttpServletResponse httpServletResponse) throws IOException; } diff --git a/src/main/java/org/mifos/processor/bulk/API/implementation/BatchTransactionsController.java b/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java similarity index 79% rename from src/main/java/org/mifos/processor/bulk/API/implementation/BatchTransactionsController.java rename to src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java index 26a68cdd..6a34612c 100644 --- a/src/main/java/org/mifos/processor/bulk/API/implementation/BatchTransactionsController.java +++ b/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java @@ -1,8 +1,22 @@ package org.mifos.processor.bulk.api.implementation; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PROGRAM_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_CLIENT_CORRELATION_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_PLATFORM_TENANT_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_TYPE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; + import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.camel.Exchange; @@ -22,25 +36,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; -import java.nio.charset.Charset; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PROGRAM_ID; -import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_CLIENT_CORRELATION_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_PLATFORM_TENANT_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_TYPE; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; @Slf4j @RestController @@ -65,25 +62,13 @@ public class BatchTransactionsController implements BatchTransactions { @SneakyThrows @Override - public String batchTransactions( - HttpServletRequest httpServletRequest, - HttpServletResponse httpServletResponse, - String requestId, - String fileName, - String purpose, - String type, - String tenant, - String registeringInstitutionId, - String programId) { + public String batchTransactions(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, String requestId, + String fileName, String purpose, String type, String tenant, String registeringInstitutionId, String programId) { log.info("Inside api logic"); - Headers.HeaderBuilder headerBuilder = new Headers.HeaderBuilder() - .addHeader(HEADER_CLIENT_CORRELATION_ID, requestId) - .addHeader(PURPOSE, purpose) - .addHeader(HEADER_TYPE, type) - .addHeader(HEADER_PLATFORM_TENANT_ID, tenant) - .addHeader(HEADER_REGISTERING_INSTITUTE_ID, registeringInstitutionId) - .addHeader(HEADER_PROGRAM_ID, programId); + Headers.HeaderBuilder headerBuilder = new Headers.HeaderBuilder().addHeader(HEADER_CLIENT_CORRELATION_ID, requestId) + .addHeader(PURPOSE, purpose).addHeader(HEADER_TYPE, type).addHeader(HEADER_PLATFORM_TENANT_ID, tenant) + .addHeader(HEADER_REGISTERING_INSTITUTE_ID, registeringInstitutionId).addHeader(HEADER_PROGRAM_ID, programId); Optional validationResponse = isValidRequest(httpServletRequest, fileName, type); if (validationResponse.isPresent()) { @@ -108,21 +93,20 @@ public String batchTransactions( String localFileName = UUID.randomUUID() + ".csv"; CsvWriter.writeToCsv(transactionList, Transaction.class, csvMapper, true, localFileName); - Headers headers = headerBuilder - .addHeader(HEADER_TYPE, "csv") - .addHeader(FILE_NAME, localFileName).build(); + Headers headers = headerBuilder.addHeader(HEADER_TYPE, "csv").addHeader(FILE_NAME, localFileName).build(); CamelApiResponse response = sendRequestToCamel(headers); httpServletResponse.setStatus(response.getStatus()); return response.getBody(); } + } @ExceptionHandler({ MultipartException.class }) public String handleMultipartException(HttpServletResponse httpServletResponse) { httpServletResponse.setStatus(httpServletResponse.SC_BAD_REQUEST); - return getErrorResponse("File not uploaded", "There was no fie uploaded with the request. " + - "Please upload a file and try again.", 400); + return getErrorResponse("File not uploaded", "There was no fie uploaded with the request. " + "Please upload a file and try again.", + 400); } private CamelApiResponse sendRequestToCamel(Headers headers) { @@ -142,31 +126,26 @@ private String getErrorResponse(String information, String description, int code } // validates the request header, and return errorJson string if the request is invalid else an empty optional - private Optional isValidRequest(HttpServletRequest httpServletRequest, - String fileName, - String type) { + private Optional isValidRequest(HttpServletRequest httpServletRequest, String fileName, String type) { - Optional response = Optional.empty(); - if ((JWSUtil.isMultipartRequest(httpServletRequest) && !type.equalsIgnoreCase("csv")) || - (!JWSUtil.isMultipartRequest(httpServletRequest) && !type.equalsIgnoreCase("raw"))) { + Optional response = Optional.empty(); + if ((JWSUtil.isMultipartRequest(httpServletRequest) && !type.equalsIgnoreCase("csv")) + || (!JWSUtil.isMultipartRequest(httpServletRequest) && !type.equalsIgnoreCase("raw"))) { String errorJson = getErrorResponse("Type mismatch", - "The value of the header \"" + HEADER_TYPE + - "\" doesn't match with the request content-type", 400); + "The value of the header \"" + HEADER_TYPE + "\" doesn't match with the request content-type", 400); response = Optional.of(errorJson); } if (JWSUtil.isMultipartRequest(httpServletRequest) && fileName.isEmpty()) { String errorJson = getErrorResponse("Header can't be empty", - "If the request is of type csv, the header \"" + - FILE_NAME + "\"can't be empty", 400); + "If the request is of type csv, the header \"" + FILE_NAME + "\"can't be empty", 400); response = Optional.of(errorJson); } if (!type.equalsIgnoreCase("raw") && !type.equalsIgnoreCase("csv")) { String errorJson = getErrorResponse("Invalid TYPE header value passed", - "The value of the header \"" + HEADER_TYPE + - "\" can be \"[raw,csv]\" but is " + type, 400); + "The value of the header \"" + HEADER_TYPE + "\" can be \"[raw,csv]\" but is " + type, 400); response = Optional.of(errorJson); } - return response; + return response; } } diff --git a/src/main/java/org/mifos/processor/bulk/API/implementation/BulkTransferController.java b/src/main/java/org/mifos/processor/bulk/api/implementation/BulkTransferController.java similarity index 99% rename from src/main/java/org/mifos/processor/bulk/API/implementation/BulkTransferController.java rename to src/main/java/org/mifos/processor/bulk/api/implementation/BulkTransferController.java index 4606a73a..f0ef69fe 100644 --- a/src/main/java/org/mifos/processor/bulk/API/implementation/BulkTransferController.java +++ b/src/main/java/org/mifos/processor/bulk/api/implementation/BulkTransferController.java @@ -1,6 +1,13 @@ package org.mifos.processor.bulk.api.implementation; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_CLIENT_CORRELATION_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_TYPE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; + import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; import org.apache.camel.Exchange; import org.apache.camel.ProducerTemplate; import org.mifos.processor.bulk.api.definition.BulkTransfer; @@ -10,12 +17,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; -import java.io.IOException; -import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_CLIENT_CORRELATION_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_TYPE; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; @RestController public class BulkTransferController implements BulkTransfer { diff --git a/src/main/java/org/mifos/processor/bulk/API/implementation/SimulateApiController.java b/src/main/java/org/mifos/processor/bulk/api/implementation/SimulateApiController.java similarity index 100% rename from src/main/java/org/mifos/processor/bulk/API/implementation/SimulateApiController.java rename to src/main/java/org/mifos/processor/bulk/api/implementation/SimulateApiController.java index 1557747c..e535c914 100644 --- a/src/main/java/org/mifos/processor/bulk/API/implementation/SimulateApiController.java +++ b/src/main/java/org/mifos/processor/bulk/api/implementation/SimulateApiController.java @@ -1,8 +1,8 @@ package org.mifos.processor.bulk.api.implementation; +import javax.servlet.http.HttpServletResponse; import org.mifos.processor.bulk.api.definition.Simulate; import org.springframework.web.bind.annotation.RestController; -import javax.servlet.http.HttpServletResponse; @RestController public class SimulateApiController implements Simulate { diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 83690f5b..0263c353 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -1,6 +1,6 @@ package org.mifos.processor.bulk.camel.config; -public class CamelProperties { +public final class CamelProperties { private CamelProperties() {} diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java b/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java index 55efee86..7a0a6b89 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java @@ -39,7 +39,7 @@ public boolean isTrusted(X509Certificate[] arg0, String arg1) throws Certificate } }).build(); } catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) { - e.printStackTrace(); + logger.debug(e.getMessage()); } clientBuilder.setSslcontext(sslContext); diff --git a/src/main/java/org/mifos/processor/bulk/camel/processor/DTOJsonConversionException.java b/src/main/java/org/mifos/processor/bulk/camel/processor/DTOJsonConversionException.java index 5f59e693..d07f4aa1 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/processor/DTOJsonConversionException.java +++ b/src/main/java/org/mifos/processor/bulk/camel/processor/DTOJsonConversionException.java @@ -2,7 +2,7 @@ public class DTOJsonConversionException extends RuntimeException { - public DTOJsonConversionException(Class dtoClass) { - super("Unable to convert " + dtoClass.getName() + "to json string"); + public DTOJsonConversionException(Class dtoClass, String message, Throwable cause) { + super(message, cause); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/processor/GsmaApiPayload.java b/src/main/java/org/mifos/processor/bulk/camel/processor/GsmaApiPayload.java index 17c8a17f..b35cfa8b 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/processor/GsmaApiPayload.java +++ b/src/main/java/org/mifos/processor/bulk/camel/processor/GsmaApiPayload.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.function.Function; +import lombok.extern.slf4j.Slf4j; import org.apache.camel.Exchange; import org.mifos.connector.common.gsma.dto.GSMATransaction; import org.mifos.processor.bulk.schema.Transaction; @@ -13,6 +14,7 @@ import org.springframework.stereotype.Component; @Component +@Slf4j public class GsmaApiPayload implements Function { @Autowired @@ -26,7 +28,7 @@ public String apply(Exchange exchange) { try { return objectMapper.writeValueAsString(gsmaTransaction); } catch (JsonProcessingException e) { - throw new DTOJsonConversionException(GSMATransaction.class); + throw new DTOJsonConversionException(GSMATransaction.class, "Unable to convert GSMATransaction to Json", e); } } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/processor/MojaloopApiPayload.java b/src/main/java/org/mifos/processor/bulk/camel/processor/MojaloopApiPayload.java index b52aee39..c3802c83 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/processor/MojaloopApiPayload.java +++ b/src/main/java/org/mifos/processor/bulk/camel/processor/MojaloopApiPayload.java @@ -25,7 +25,7 @@ public String apply(Exchange exchange) { try { return objectMapper.writeValueAsString(inboundTransferPayload); } catch (JsonProcessingException e) { - throw new DTOJsonConversionException(MojaloopApiPayload.class); + throw new DTOJsonConversionException(MojaloopApiPayload.class, "Unable to convert MojaloopPayload to JSON", e); } } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java index bd8a2862..15836143 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java @@ -1,44 +1,42 @@ package org.mifos.processor.bulk.camel.routes; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_IDENTITY; +import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYMENT_MODALITY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; + +import javax.net.ssl.HttpsURLConnection; import org.apache.camel.Exchange; import org.apache.camel.Processor; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import javax.net.ssl.HttpsURLConnection; - -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; - @Component public class AccountLookupRoute extends BaseRouteBuilder { + @Value("${identity_account_mapper.account_lookup}") private String accountLookupEndpoint; @Value("${identity_account_mapper.hostname}") private String identityURL; + @Override public void configure() throws Exception { Processor disableSslProcessor = new Processor() { + @Override public void process(Exchange exchange) throws Exception { // Disable SSL certificate validation HttpsURLConnection.setDefaultHostnameVerifier((hostname, sslSession) -> true); } }; - from("direct:send-account-lookup") - .id("account-lookup") - .process(exchange -> { - String callbackUrl = exchange.getProperty(CALLBACK, String.class); - String registeringInstitutionId = exchange.getProperty(HEADER_REGISTERING_INSTITUTE_ID, String.class); - exchange.getIn().setHeader(CALLBACK, callbackUrl); - exchange.getIn().setHeader(HEADER_REGISTERING_INSTITUTE_ID, registeringInstitutionId); - }) - .setHeader(Exchange.HTTP_METHOD, constant("GET")) - .toD(identityURL + accountLookupEndpoint + "?" - + PAYEE_IDENTITY + "=${exchangeProperty.payeeIdentity}&" - + PAYMENT_MODALITY + "=${exchangeProperty.paymentModality}&" - + "requestId=${exchangeProperty.requestId}") - .log("API Response: ${body}") - .process(disableSslProcessor); + from("direct:send-account-lookup").id("account-lookup").process(exchange -> { + String callbackUrl = exchange.getProperty(CALLBACK, String.class); + String registeringInstitutionId = exchange.getProperty(HEADER_REGISTERING_INSTITUTE_ID, String.class); + exchange.getIn().setHeader(CALLBACK, callbackUrl); + exchange.getIn().setHeader(HEADER_REGISTERING_INSTITUTE_ID, registeringInstitutionId); + }).setHeader(Exchange.HTTP_METHOD, constant("GET")) + .toD(identityURL + accountLookupEndpoint + "?" + PAYEE_IDENTITY + "=${exchangeProperty.payeeIdentity}&" + PAYMENT_MODALITY + + "=${exchangeProperty.paymentModality}&" + "requestId=${exchangeProperty.requestId}") + .log("API Response: ${body}").process(disableSslProcessor); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java index b51586eb..5d597060 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java @@ -1,6 +1,8 @@ package org.mifos.processor.bulk.camel.routes; -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_STATUS_FAILED; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.OPS_APP_ACCESS_TOKEN; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_RATE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_CODE; @@ -32,19 +34,13 @@ public void configure() throws Exception { * Uploads the updated file in cloud. */ from(RouteId.BATCH_AGGREGATE.getValue()).id(RouteId.BATCH_AGGREGATE.getValue()) - .log("Starting route " + RouteId.BATCH_AGGREGATE.name()) - .to("direct:get-access-token") - .choice() + .log("Starting route " + RouteId.BATCH_AGGREGATE.name()).to("direct:get-access-token").choice() .when(exchange -> exchange.getProperty(OPS_APP_ACCESS_TOKEN, String.class) != null) - .log(LoggingLevel.INFO, "Got access token, moving on to API call") - .to("direct:batch-aggregate-api-call") - .to("direct:batch-aggregate-response-handler") - .otherwise() - .log(LoggingLevel.INFO, "Authentication failed.") - .endChoice(); + .log(LoggingLevel.INFO, "Got access token, moving on to API call").to("direct:batch-aggregate-api-call") + .to("direct:batch-aggregate-response-handler").otherwise().log(LoggingLevel.INFO, "Authentication failed.").endChoice(); getBaseExternalApiRequestRouteDefinition("batch-aggregate-api-call", HttpRequestMethod.GET) -// .setHeader(Exchange.REST_HTTP_QUERY, simple("batchId=${exchangeProperty." + BATCH_ID + "}")) + // .setHeader(Exchange.REST_HTTP_QUERY, simple("batchId=${exchangeProperty." + BATCH_ID + "}")) .setHeader("Authorization", simple("Bearer ${exchangeProperty." + OPS_APP_ACCESS_TOKEN + "}")) .setHeader(HEADER_PLATFORM_TENANT_ID, simple("${exchangeProperty." + TENANT_ID + "}")).process(exchange -> { logger.info(exchange.getIn().getHeaders().toString()); @@ -52,8 +48,7 @@ public void configure() throws Exception { .log(LoggingLevel.DEBUG, "Batch aggregate API response: \n\n ${body}") .log(LoggingLevel.INFO, "Aggregate Response body: ${body}"); - from("direct:batch-aggregate-response-handler") - .id("direct:batch-aggregate-response-handler") + from("direct:batch-aggregate-response-handler").id("direct:batch-aggregate-response-handler") .log("Starting route direct:batch-aggregate-response-handler") // .setBody(exchange -> exchange.getIn().getBody(String.class)) .choice().when(header("CamelHttpResponseCode").isEqualTo("200")).log(LoggingLevel.INFO, "Batch summary request successful") diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java index e0e7e53d..74cbb943 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java @@ -9,13 +9,13 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FAILED_AMOUNT; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ONGOING_AMOUNT; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TOTAL_AMOUNT; + import com.fasterxml.jackson.databind.MappingIterator; import com.fasterxml.jackson.databind.SequenceWriter; import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.dataformat.csv.CsvSchema; import java.io.File; import java.io.FileReader; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import org.mifos.processor.bulk.schema.Transaction; diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java index 0c332f0a..c4d48b78 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileRoute.java @@ -5,7 +5,6 @@ import java.io.File; import java.io.FileOutputStream; -import java.io.InputStream; import org.mifos.processor.bulk.file.FileTransferService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/HealthRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/HealthRoute.java new file mode 100644 index 00000000..e7a61686 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/HealthRoute.java @@ -0,0 +1,19 @@ +package org.mifos.processor.bulk.camel.routes; + +import org.json.JSONObject; +import org.springframework.stereotype.Component; + +@Component +public class HealthRoute extends BaseRouteBuilder { + + @Override + public void configure() throws Exception { + + // todo remove once camel APIs are migrated to spring + from("rest:GET:/actuator/health/liveness").id("rest:GET:/actuator/health/liveness").setBody(exchange -> { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("status", "UP"); + return jsonObject.toString(); + }); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index df306496..49e7c38a 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -1,7 +1,21 @@ package org.mifos.processor.bulk.camel.routes; -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_ID_HEADER; +import static org.mifos.processor.bulk.camel.config.CamelProperties.EXTERNAL_ENDPOINT; +import static org.mifos.processor.bulk.camel.config.CamelProperties.EXTERNAL_ENDPOINT_FAILED; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_CLIENT_CORRELATION_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.IS_PAYMENT_MODE_VALID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; +import static org.mifos.processor.bulk.camel.config.CamelProperties.OVERRIDE_HEADER; +import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYMENT_MODE_TYPE; +import static org.mifos.processor.bulk.camel.config.CamelProperties.RESULT_TRANSACTION_LIST; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_ELEMENT; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_LENGTH; +import static org.mifos.processor.bulk.camel.config.CamelProperties.ZEEBE_VARIABLE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETED_AMOUNT; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DEBULKINGDFSPID; @@ -63,12 +77,8 @@ public void configure() throws Exception { * Builds the [Transaction] array using [direct:get-transaction-array] route. 3. Loops through each transaction * and start the respective workflow */ - from(RouteId.INIT_SUB_BATCH.getValue()) - .id(RouteId.INIT_SUB_BATCH.getValue()) - .log("Starting route " + RouteId.INIT_SUB_BATCH.name()) - .to("direct:download-file") - .to("direct:get-transaction-array") - .to("direct:start-workflow-step1"); + from(RouteId.INIT_SUB_BATCH.getValue()).id(RouteId.INIT_SUB_BATCH.getValue()).log("Starting route " + RouteId.INIT_SUB_BATCH.name()) + .to("direct:download-file").to("direct:get-transaction-array").to("direct:start-workflow-step1"); // crates the zeebe variables map and starts the workflow by calling >> direct:start-workflow-step2 from("direct:start-workflow-step1").id("direct:start-flow-step1").log("Starting route direct:start-flow-step1") @@ -120,11 +130,10 @@ public void configure() throws Exception { int index = exchange.getProperty(Exchange.LOOP_INDEX, Integer.class); List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); Transaction transaction = transactionList.get(index); - exchange.setProperty(REQUEST_ID, transaction.getRequestId()); - logger.info("REQUEST_ID: {}", transaction.getRequestId()); + exchange.setProperty(REQUEST_ID, transaction.getRequestId()); + logger.info("REQUEST_ID: {}", transaction.getRequestId()); exchange.setProperty(TRANSACTION_LIST_ELEMENT, transaction); - }).setHeader("Platform-TenantId", exchangeProperty(TENANT_NAME)) - .to("direct:dynamic-payload-setter") + }).setHeader("Platform-TenantId", exchangeProperty(TENANT_NAME)).to("direct:dynamic-payload-setter") .to("direct:external-api-call").to("direct:external-api-response-handler").end() // end loop block .endChoice(); @@ -189,6 +198,7 @@ public void configure() throws Exception { log.debug("Variables: {}", exchange.getProperties()); log.debug("Emergency: {}", exchange.getIn().getHeaders()); }) + .toD(channelURL + "${exchangeProperty.extEndpoint}" + "?bridgeEndpoint=true&throwExceptionOnFailure=false") .log(LoggingLevel.DEBUG, "Response body: ${body}").otherwise().endChoice(); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java index 71c2c565..5f23e11c 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java @@ -4,7 +4,6 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERED_BY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERING_FAILED; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import org.mifos.processor.bulk.schema.Transaction; @@ -68,12 +67,8 @@ public void configure() { if (stringListHashMap.containsKey(key)) { stringListHashMap.get(key).add(transaction); } else { - stringListHashMap.put(key, new ArrayList() { - - { - add(transaction); - } - }); + transactionList.add(transaction); + stringListHashMap.put(key, transactionList); } }); transactionList.clear(); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 1c220823..5b7ebda5 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -1,13 +1,21 @@ package org.mifos.processor.bulk.camel.routes; -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; +import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_REQUEST_TYPE; import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_CLIENT_CORRELATION_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; -import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_REQUEST_TYPE; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PROGRAM_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.IS_UPDATED; +import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; +import static org.mifos.processor.bulk.camel.config.CamelProperties.OVERRIDE_HEADER; +import static org.mifos.processor.bulk.camel.config.CamelProperties.PROGRAM_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.REGISTERING_INSTITUTE_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.RESULT_TRANSACTION_LIST; import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.APPROVAL_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.AUTHORIZATION_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_AGGREGATE_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BULK_NOTIF_FAILURE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BULK_NOTIF_SUCCESS; @@ -17,19 +25,25 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_THRESHOLD_CHECK_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FORMATTING_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.IS_FILE_VALID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_CALLBACK_RETRY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_STATUS_RETRY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.NOTE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERING_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYER_IDENTIFIER_TYPE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYER_IDENTIFIER_VALUE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASES; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASE_COUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PROGRAM_NAME; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.THRESHOLD_DELAY; + +import com.fasterxml.jackson.core.JsonProcessingException; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -37,13 +51,12 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.ArrayList; -import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; import org.apache.tika.Tika; @@ -141,19 +154,15 @@ private void setup() { exchange.setProperty(PURPOSE, purpose); }).wireTap("direct:start-batch-process-csv").to("direct:pollingOutput"); - from("direct:validate-tenant") - .id("direct:validate-tenant") - .log("Validating tenant") - .process(exchange -> { - String tenantName = exchange.getIn().getHeader(HEADER_PLATFORM_TENANT_ID, String.class); - // validation is disabled for now - /*if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { - throw new Exception("Invalid tenant value."); - }*/ - exchange.setProperty(TENANT_NAME, tenantName); - }) - .setHeader("Content-Type", constant("application/json;charset=UTF-8")) - .log("Completed route direct:validate-tenant"); + from("direct:validate-tenant").id("direct:validate-tenant").log("Validating tenant").process(exchange -> { + String tenantName = exchange.getIn().getHeader(HEADER_PLATFORM_TENANT_ID, String.class); + // validation is disabled for now + /* + * if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { throw new + * Exception("Invalid tenant value."); } + */ + exchange.setProperty(TENANT_NAME, tenantName); + }).setHeader("Content-Type", constant("application/json;charset=UTF-8")).log("Completed route direct:validate-tenant"); // this route is responsible for editing the incoming records based on configuration // this step is done to make sure the file format of CSV is not altered and only the data is updated based on @@ -178,7 +187,8 @@ private void setup() { RegisteringInstitutionConfig registeringInstitutionConfig = budgetAccountConfig .getByRegisteringInstituteId(registeringInstituteId); if (registeringInstitutionConfig == null) { - logger.debug("Element in nested in config: {}", budgetAccountConfig.getRegisteringInstitutions().get(0).getPrograms().size()); + logger.debug("Element in nested in config: {}", + budgetAccountConfig.getRegisteringInstitutions().get(0).getPrograms().size()); logger.debug("Registering institute id is null"); exchange.setProperty(IS_UPDATED, false); @@ -213,9 +223,7 @@ private void setup() { .when(exchange -> exchange.getProperty(IS_UPDATED, Boolean.class)) // warning: changing this flag can break things .setProperty(OVERRIDE_HEADER, constant(true)) // default header in CSV file will be used - .to("direct:update-file-v2") - .otherwise() - .log(LoggingLevel.INFO, "No update"); + .to("direct:update-file-v2").otherwise().log(LoggingLevel.INFO, "No update"); from("direct:start-batch-process-csv").id("direct:start-batch-process-csv").log("Starting route direct:start-batch-process-csv") .to("direct:update-incoming-data").process(exchange -> { @@ -235,7 +243,7 @@ private void setup() { File file = new File(fileName); file.setWritable(true); file.setReadable(true); - + logger.debug("File absolute path: {}", file.getAbsolutePath()); boolean verifyData = verifyData(file); @@ -272,7 +280,7 @@ private void setup() { variables.put(IS_FILE_VALID, true); setConfigProperties(variables); - logger.debug("Zeebe variables published: {}", variables); + logger.debug("Zeebe variables published: {}", variables); log.debug("Variables published to zeebe: {}", variables); JSONObject response = new JSONObject(); @@ -324,9 +332,7 @@ private void setup() { exchange.setProperty(CLIENT_CORRELATION_ID, clientCorrelationId); exchange.setProperty(REGISTERING_INSTITUTE_ID, registeringInstitutionId); exchange.setProperty(PROGRAM_ID, programId); - }) - .choice() - .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("raw")) + }).choice().when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("raw")) .to("direct:start-batch-process-raw") .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("csv")) .to("direct:start-batch-process-csv").otherwise() diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java b/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java index b4bede34..82309c7d 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java @@ -2,14 +2,13 @@ public enum RouteId { - PARTY_LOOKUP("direct:partyLookup"), APPROVAL("direct:approval"), ORDERING("direct:ordering"), - SPLITTING("direct:splitting"), FORMATTING("direct:formatting"), BATCH_STATUS("direct:batchStatus"), - SEND_CALLBACK("direct:sendCallback"), MERGE_BACK("direct:mergeSubBatch"), - INIT_SUB_BATCH("direct:init-sub-batches"), BATCH_AGGREGATE("direct:batch-aggregate"); + PARTY_LOOKUP("direct:partyLookup"), APPROVAL("direct:approval"), ORDERING("direct:ordering"), SPLITTING("direct:splitting"), FORMATTING( + "direct:formatting"), BATCH_STATUS("direct:batchStatus"), SEND_CALLBACK("direct:sendCallback"), MERGE_BACK( + "direct:mergeSubBatch"), INIT_SUB_BATCH("direct:init-sub-batches"), BATCH_AGGREGATE("direct:batch-aggregate"); private final String value; - private RouteId(String s) { + RouteId(String s) { value = s; } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java index 5caff8ac..6810ca02 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java @@ -39,12 +39,8 @@ public void configure() throws Exception { exchange.getProperty(COMPLETION_RATE).toString()); callbackUrl = exchange.getProperty(CALLBACK_URL).toString(); return body; - }) - .to("direct:set-jws-signature") - .toD("${header.callbackUrl}?bridgeEndpoint=true&throwExceptionOnFailure=false") - .choice() - .when(header("CamelHttpResponseCode").startsWith("2")) - .log(LoggingLevel.INFO, "Callback sending was successful") + }).to("direct:set-jws-signature").toD("${header.callbackUrl}?bridgeEndpoint=true&throwExceptionOnFailure=false").choice() + .when(header("CamelHttpResponseCode").startsWith("2")).log(LoggingLevel.INFO, "Callback sending was successful") .process(exchange -> { List phases = (List) exchange.getProperty(PHASES); @@ -80,7 +76,9 @@ public void configure() throws Exception { public void eliminatePhases(Exchange exchange, List phases, int phaseCount, int completionRate) { int i = 0; while (phases.size() > 0 && phases.size() > i) { - if (phases.get(i) <= completionRate) phases.remove(i); + if (phases.get(i) <= completionRate) { + phases.remove(i); + } i++; } exchange.setProperty(PHASES, phases); diff --git a/src/main/java/org/mifos/processor/bulk/config/BudgetAccountConfig.java b/src/main/java/org/mifos/processor/bulk/config/BudgetAccountConfig.java index 0bb7930e..8b22c615 100644 --- a/src/main/java/org/mifos/processor/bulk/config/BudgetAccountConfig.java +++ b/src/main/java/org/mifos/processor/bulk/config/BudgetAccountConfig.java @@ -1,11 +1,11 @@ package org.mifos.processor.bulk.config; +import java.util.ArrayList; +import java.util.List; import lombok.Getter; import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; -import java.util.ArrayList; -import java.util.List; @Getter @Setter @@ -16,9 +16,6 @@ public class BudgetAccountConfig { private List registeringInstitutions = new ArrayList<>(); public RegisteringInstitutionConfig getByRegisteringInstituteId(String id) { - return getRegisteringInstitutions().stream() - .filter(p -> p.getId().equals(id)) - .findFirst() - .orElse(null); + return getRegisteringInstitutions().stream().filter(p -> p.getId().equals(id)).findFirst().orElse(null); } } diff --git a/src/main/java/org/mifos/processor/bulk/config/Program.java b/src/main/java/org/mifos/processor/bulk/config/Program.java index f9a4a674..46bd2e07 100644 --- a/src/main/java/org/mifos/processor/bulk/config/Program.java +++ b/src/main/java/org/mifos/processor/bulk/config/Program.java @@ -11,6 +11,9 @@ @NoArgsConstructor public class Program { - private String id, name, identifierType, identifierValue; + private String id; + private String name; + private String identifierType; + private String identifierValue; } diff --git a/src/main/java/org/mifos/processor/bulk/config/RegisteringInstitutionConfig.java b/src/main/java/org/mifos/processor/bulk/config/RegisteringInstitutionConfig.java index fadb404b..cb71a9f7 100644 --- a/src/main/java/org/mifos/processor/bulk/config/RegisteringInstitutionConfig.java +++ b/src/main/java/org/mifos/processor/bulk/config/RegisteringInstitutionConfig.java @@ -1,11 +1,11 @@ package org.mifos.processor.bulk.config; +import java.util.ArrayList; +import java.util.List; import lombok.Getter; import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; -import java.util.ArrayList; -import java.util.List; @Getter @Setter @@ -17,10 +17,7 @@ public class RegisteringInstitutionConfig { private List programs = new ArrayList<>(); public Program getByProgramId(String id) { - return getPrograms().stream() - .filter(p -> p.getId().equals(id)) - .findFirst() - .orElse(null); + return getPrograms().stream().filter(p -> p.getId().equals(id)).findFirst().orElse(null); } } diff --git a/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java b/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java index 571c2705..ec9dcd56 100644 --- a/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java +++ b/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java @@ -4,11 +4,11 @@ import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; +import com.amazonaws.util.IOUtils; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import com.amazonaws.util.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -33,7 +33,7 @@ public byte[] downloadFile(String fileName, String bucketName) { byte[] content = IOUtils.toByteArray(inputStream); return content; } catch (IOException e) { - e.printStackTrace(); + logger.debug("{}", e.getMessage()); } return null; } diff --git a/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java b/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java index 43940886..33df0fa3 100644 --- a/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java +++ b/src/main/java/org/mifos/processor/bulk/file/AzureFileTransferImpl.java @@ -1,14 +1,13 @@ package org.mifos.processor.bulk.file; import com.azure.storage.blob.BlobClientBuilder; +import com.azure.storage.blob.models.BlobProperties; +import com.azure.storage.blob.specialized.BlobInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Paths; - -import com.azure.storage.blob.models.BlobProperties; -import com.azure.storage.blob.specialized.BlobInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -36,7 +35,7 @@ public byte[] downloadFile(String fileName, String bucketName) { temp.delete(); return content; } catch (Exception e) { - e.printStackTrace(); + logger.debug("{}", e.getMessage()); } return null; } @@ -76,7 +75,7 @@ public InputStream streamFile(String fileName, String bucketName) { // temp.delete(); return csvInputStream; } catch (Exception e) { - e.printStackTrace(); + logger.debug(e.getMessage()); } return null; } diff --git a/src/main/java/org/mifos/processor/bulk/file/FileStorageService.java b/src/main/java/org/mifos/processor/bulk/file/FileStorageService.java index 42967c3c..01682f1f 100644 --- a/src/main/java/org/mifos/processor/bulk/file/FileStorageService.java +++ b/src/main/java/org/mifos/processor/bulk/file/FileStorageService.java @@ -1,14 +1,13 @@ package org.mifos.processor.bulk.file; +import java.io.InputStream; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; -import java.io.InputStream; - @Service public interface FileStorageService { - public String save(MultipartFile file); + String save(MultipartFile file); String save(InputStream file, String filename); diff --git a/src/main/java/org/mifos/processor/bulk/file/FileStorageServiceImpl.java b/src/main/java/org/mifos/processor/bulk/file/FileStorageServiceImpl.java index bc7ad934..2196cf21 100644 --- a/src/main/java/org/mifos/processor/bulk/file/FileStorageServiceImpl.java +++ b/src/main/java/org/mifos/processor/bulk/file/FileStorageServiceImpl.java @@ -1,8 +1,5 @@ package org.mifos.processor.bulk.file; -import org.springframework.stereotype.Service; -import org.springframework.web.multipart.MultipartFile; - import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; @@ -10,6 +7,8 @@ import java.nio.file.Paths; import java.util.Objects; import java.util.UUID; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; @Service public class FileStorageServiceImpl implements FileStorageService { @@ -22,7 +21,7 @@ public String save(MultipartFile file) { try { Files.copy(file.getInputStream(), this.root.resolve(filename)); } catch (Exception e) { - throw new RuntimeException(e.getMessage()); + throw new RuntimeException("Failed to save file: " + e.getMessage(), e); } return filename; } @@ -33,7 +32,7 @@ public String save(InputStream inputStream, String filename) { try { Files.copy(inputStream, this.root.resolve(uniqueFileName)); } catch (Exception e) { - throw new RuntimeException(e.getMessage()); + throw new RuntimeException("Failed to save file: " + e.getMessage(), e); } return uniqueFileName; } diff --git a/src/main/java/org/mifos/processor/bulk/format/RestRequestConvertor.java b/src/main/java/org/mifos/processor/bulk/format/RestRequestConvertor.java index cef808ad..684635ef 100644 --- a/src/main/java/org/mifos/processor/bulk/format/RestRequestConvertor.java +++ b/src/main/java/org/mifos/processor/bulk/format/RestRequestConvertor.java @@ -1,15 +1,14 @@ package org.mifos.processor.bulk.format; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; import org.mifos.processor.bulk.schema.BatchRequestDTO; import org.mifos.processor.bulk.schema.Party; import org.mifos.processor.bulk.schema.PartyType; import org.mifos.processor.bulk.schema.Transaction; import org.springframework.stereotype.Component; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - @Component public class RestRequestConvertor implements EntityMapper { diff --git a/src/main/java/org/mifos/processor/bulk/format/StandardValue.java b/src/main/java/org/mifos/processor/bulk/format/StandardValue.java index 9d86d9c8..d0037157 100644 --- a/src/main/java/org/mifos/processor/bulk/format/StandardValue.java +++ b/src/main/java/org/mifos/processor/bulk/format/StandardValue.java @@ -1,6 +1,8 @@ package org.mifos.processor.bulk.format; -public class StandardValue { +public final class StandardValue { + + private StandardValue() {} public static final String GSMA = "GSMA"; diff --git a/src/main/java/org/mifos/processor/bulk/format/helper/BaseMapper.java b/src/main/java/org/mifos/processor/bulk/format/helper/BaseMapper.java index ef311429..bae65290 100644 --- a/src/main/java/org/mifos/processor/bulk/format/helper/BaseMapper.java +++ b/src/main/java/org/mifos/processor/bulk/format/helper/BaseMapper.java @@ -4,11 +4,11 @@ import java.util.List; import org.mifos.processor.bulk.schema.CsvSchema; -public abstract class BaseMapper implements Mapper { +public abstract class BaseMapper implements Mapper { @Override - public List convertList(List objects) { - List list = new ArrayList<>(); + public List convertList(List objects) { + List list = new ArrayList<>(); objects.forEach(transaction -> { list.add(convert(transaction)); }); diff --git a/src/main/java/org/mifos/processor/bulk/format/helper/Mapper.java b/src/main/java/org/mifos/processor/bulk/format/helper/Mapper.java index ba259dc0..e6743117 100644 --- a/src/main/java/org/mifos/processor/bulk/format/helper/Mapper.java +++ b/src/main/java/org/mifos/processor/bulk/format/helper/Mapper.java @@ -12,7 +12,7 @@ public interface Mapper { * of type [FROM] * @return object of type [To] */ - public TO convert(FROM object); + TO convert(FROM object); /** * Use for converting the list of objects of type [FROM] to an object of type [TO] @@ -21,6 +21,6 @@ public interface Mapper { * lost of object of type [FROM] * @return objects of type [To] */ - public List convertList(List objects); + List convertList(List objects); } diff --git a/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java b/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java index b98386cf..4212c75c 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java +++ b/src/main/java/org/mifos/processor/bulk/schema/BatchDTO.java @@ -1,21 +1,20 @@ package org.mifos.processor.bulk.schema; +import java.math.BigDecimal; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; -import java.math.BigDecimal; - @Getter @Setter @AllArgsConstructor @NoArgsConstructor public class BatchDTO { - private String batch_id; + private String batchId; - private String request_id; + private String requestId; private Long total; @@ -37,7 +36,7 @@ public class BatchDTO { private String notes; - private String created_at; + private String createdAt; private String status; diff --git a/src/main/java/org/mifos/processor/bulk/schema/BatchRequestDTO.java b/src/main/java/org/mifos/processor/bulk/schema/BatchRequestDTO.java index c0ce1cc1..85d1551d 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/BatchRequestDTO.java +++ b/src/main/java/org/mifos/processor/bulk/schema/BatchRequestDTO.java @@ -1,20 +1,23 @@ package org.mifos.processor.bulk.schema; +import java.util.List; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; -import java.util.List; - @Getter @Setter @AllArgsConstructor @NoArgsConstructor public class BatchRequestDTO { - List creditParty, debitParty; + List creditParty; + List debitParty; - String subType, amount, currency, descriptionText; + String subType; + String amount; + String currency; + String descriptionText; } diff --git a/src/main/java/org/mifos/processor/bulk/schema/Party.java b/src/main/java/org/mifos/processor/bulk/schema/Party.java index 92fdb5a1..a795fc78 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/Party.java +++ b/src/main/java/org/mifos/processor/bulk/schema/Party.java @@ -11,6 +11,7 @@ @NoArgsConstructor public class Party { - String key, value; + String key; + String value; } diff --git a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java index 3c56a519..e49eb3e5 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java +++ b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java @@ -1,9 +1,9 @@ package org.mifos.processor.bulk.schema; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; import lombok.Getter; import lombok.Setter; diff --git a/src/main/java/org/mifos/processor/bulk/utility/CsvWriter.java b/src/main/java/org/mifos/processor/bulk/utility/CsvWriter.java index 95694fce..9012cded 100644 --- a/src/main/java/org/mifos/processor/bulk/utility/CsvWriter.java +++ b/src/main/java/org/mifos/processor/bulk/utility/CsvWriter.java @@ -3,12 +3,13 @@ import com.fasterxml.jackson.databind.SequenceWriter; import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.dataformat.csv.CsvSchema; - import java.io.File; import java.io.IOException; import java.util.List; -public class CsvWriter { +public final class CsvWriter { + + private CsvWriter() {} public static void writeToCsv(List data, Class tClass, CsvMapper csvMapper, boolean overrideHeader, String filepath) throws IOException { diff --git a/src/main/java/org/mifos/processor/bulk/utility/Headers.java b/src/main/java/org/mifos/processor/bulk/utility/Headers.java index 69aa9f0f..03bff690 100644 --- a/src/main/java/org/mifos/processor/bulk/utility/Headers.java +++ b/src/main/java/org/mifos/processor/bulk/utility/Headers.java @@ -4,11 +4,11 @@ import java.util.Map; import java.util.Set; -public class Headers { +public final class Headers { + private Map headers; - private Headers() { - } + private Headers() {} private void setHeaders(Map headers) { this.headers = headers; @@ -27,6 +27,7 @@ public Object get(String key) { } public static class HeaderBuilder { + private Map headers = new HashMap<>(); public HeaderBuilder addHeader(String key, Object value) { diff --git a/src/main/java/org/mifos/processor/bulk/utility/SpringWrapperUtil.java b/src/main/java/org/mifos/processor/bulk/utility/SpringWrapperUtil.java index 8b77e818..ca60bcb9 100644 --- a/src/main/java/org/mifos/processor/bulk/utility/SpringWrapperUtil.java +++ b/src/main/java/org/mifos/processor/bulk/utility/SpringWrapperUtil.java @@ -3,11 +3,12 @@ import org.apache.camel.CamelContext; import org.apache.camel.Exchange; import org.apache.camel.support.DefaultExchange; -import org.springframework.web.multipart.MultipartFile; -public class SpringWrapperUtil { - public static Exchange getDefaultWrappedExchange(CamelContext camelContext, - Headers headers) { +public final class SpringWrapperUtil { + + private SpringWrapperUtil() {} + + public static Exchange getDefaultWrappedExchange(CamelContext camelContext, Headers headers) { Exchange exchange = new DefaultExchange(camelContext); // Setting headers diff --git a/src/main/java/org/mifos/processor/bulk/utility/Utils.java b/src/main/java/org/mifos/processor/bulk/utility/Utils.java index b9cc5c4e..b896bc9d 100644 --- a/src/main/java/org/mifos/processor/bulk/utility/Utils.java +++ b/src/main/java/org/mifos/processor/bulk/utility/Utils.java @@ -5,6 +5,7 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import lombok.extern.slf4j.Slf4j; import org.mifos.connector.common.channel.dto.TransactionChannelRequestDTO; import org.mifos.connector.common.gsma.dto.Fee; import org.mifos.connector.common.gsma.dto.GSMATransaction; @@ -21,7 +22,10 @@ import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.schema.TransactionResult; -public class Utils { +@Slf4j +public final class Utils { + + private Utils() {} public static String getTenantSpecificWorkflowId(String originalWorkflowName, String tenantName) { return originalWorkflowName.replace("{dfspid}", tenantName); @@ -50,7 +54,7 @@ public static String mergeCsvFile(String file1, String file2) { in.close(); out.close(); } catch (IOException e) { - e.printStackTrace(); + log.debug(e.getMessage()); return null; } @@ -201,13 +205,10 @@ public static GSMATransaction convertTxnToGSMA(Transaction transaction) { public static TransactionChannelRequestDTO convertTxnToInboundTransferPayload(Transaction transaction) { TransactionChannelRequestDTO requestDTO = new TransactionChannelRequestDTO(); - requestDTO.setAmount(new MoneyData() { - - { - setCurrency(transaction.getCurrency()); - setAmount(transaction.getAmount()); - } - }); + MoneyData moneyData = new MoneyData(); + moneyData.setCurrency(transaction.getCurrency()); + moneyData.setAmount(transaction.getAmount()); + requestDTO.setAmount(moneyData); IdentifierType identifierType; try { diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeMessages.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeMessages.java index c4538776..ee6e9fb7 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeMessages.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeMessages.java @@ -1,6 +1,6 @@ package org.mifos.processor.bulk.zeebe; -public class ZeebeMessages { +public final class ZeebeMessages { private ZeebeMessages() {} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 57bd89f5..8a1d3e94 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -1,6 +1,6 @@ package org.mifos.processor.bulk.zeebe; -public class ZeebeVariables { +public final class ZeebeVariables { private ZeebeVariables() {} @@ -149,12 +149,12 @@ private ZeebeVariables() {} public static final String AUTHORIZATION_FAIL_REASON = "authorizationFailReason"; - public static final String PAYER_IDENTIFIER = "payerIdentifier"; + public static final String PAYER_IDENTIFIER = "payerIdentifier"; - public static final String PARTY_LOOKUP_SUCCESSFUL_TRANSACTION_COUNT = "partyLookupSuccessfulTransactionAmount"; + public static final String PARTY_LOOKUP_SUCCESSFUL_TRANSACTION_COUNT = "partyLookupSuccessfulTransactionAmount"; - public static final String CURRENCY = "currency"; + public static final String CURRENCY = "currency"; - public static final String AUTHORIZATION_RESPONSE = "authorizationResponse"; + public static final String AUTHORIZATION_RESPONSE = "authorizationResponse"; } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java index cec0c5d4..3805f907 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeWorkers.java @@ -74,7 +74,8 @@ private void workerBulkProcessor() { InputStream csvFileInputStream = fileTransferService.streamFile(fileName, bucketName); CsvSchema schema = CsvSchema.emptySchema().withHeader(); - MappingIterator readValues = csvMapper.readerWithSchemaFor(Transaction.class).with(schema).readValues(csvFileInputStream); + MappingIterator readValues = csvMapper.readerWithSchemaFor(Transaction.class).with(schema) + .readValues(csvFileInputStream); /* * while (readValues.hasNext()) { Transaction current = readValues.next(); current.setBatchId(batchId); if diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java index 25a96055..a424dc2f 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupCallbackWorker.java @@ -1,7 +1,9 @@ package org.mifos.processor.bulk.zeebe.worker; import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_PARTY_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CHANNEL_REQUEST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORIGIN_CHANNEL_REQUEST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_FSP_ID; import static org.mifos.processor.bulk.zeebe.worker.Worker.ACCOUNT_LOOKUP_CALLBACK; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java index 72962261..d682dabb 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java @@ -1,10 +1,10 @@ package org.mifos.processor.bulk.zeebe.worker; import static org.mifos.processor.bulk.camel.config.CamelProperties.CACHED_TRANSACTION_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.HOST; import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_IDENTITY; import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYMENT_MODALITY; -import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ACCOUNT_LOOKUP_RETRY_COUNT; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CHANNEL_REQUEST; @@ -18,19 +18,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.camunda.zeebe.client.ZeebeClient; import java.util.Map; - import org.apache.camel.CamelContext; import org.apache.camel.Exchange; import org.apache.camel.ProducerTemplate; import org.apache.camel.support.DefaultExchange; import org.mifos.connector.common.channel.dto.TransactionChannelRequestDTO; import org.mifos.connector.common.mojaloop.dto.PartyIdInfo; -import org.mifos.processor.bulk.schema.AuthorizationRequest; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.*; import org.springframework.stereotype.Component; -import org.springframework.web.client.RestTemplate; @Component public class AccountLookupWorker extends BaseWorker { @@ -87,7 +83,6 @@ public void setup() { exchange.setProperty(HEADER_REGISTERING_INSTITUTE_ID, existingVariables.get(HEADER_REGISTERING_INSTITUTE_ID)); producerTemplate.send("direct:send-account-lookup", exchange); - client.newCompleteCommand(job.getKey()).variables(existingVariables).send(); }).name(String.valueOf(ACCOUNT_LOOKUP)).open(); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java index 5f437f3e..c281de23 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java @@ -19,7 +19,7 @@ public class AggregateWorker extends BaseWorker { @Override public void setup() { -// newWorker(Worker.BATCH_STATUS, (client, job) -> { + // newWorker(Worker.BATCH_STATUS, (client, job) -> { newWorker(Worker.BATCH_AGGREGATE, (client, job) -> { logger.info("Started batchAggregateWorker"); logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AuthorizationWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AuthorizationWorker.java index fd08ba5b..b8f22132 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AuthorizationWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AuthorizationWorker.java @@ -1,11 +1,15 @@ package org.mifos.processor.bulk.zeebe.worker; -import java.util.HashMap; -import java.util.Map; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.APPROVED_AMOUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.AUTHORIZATION_ACCEPTED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.AUTHORIZATION_SUCCESSFUL; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CLIENT_CORRELATION_ID; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.micrometer.core.instrument.util.StringUtils; +import java.util.Map; import org.mifos.processor.bulk.schema.AuthorizationRequest; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -17,12 +21,6 @@ import org.springframework.stereotype.Component; import org.springframework.web.client.RestTemplate; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.AUTHORIZATION_SUCCESSFUL; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.APPROVED_AMOUNT; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CLIENT_CORRELATION_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.AUTHORIZATION_ACCEPTED; - @Component public class AuthorizationWorker extends BaseWorker { @@ -75,7 +73,8 @@ public void setup() { }); } - private HttpStatus invokeBatchAuthorizationApi(String batchId, AuthorizationRequest requestPayload, String clientCorrelationId) throws JsonProcessingException { + private HttpStatus invokeBatchAuthorizationApi(String batchId, AuthorizationRequest requestPayload, String clientCorrelationId) + throws JsonProcessingException { logger.info("Calling auth API"); if (StringUtils.isBlank(requestPayload.getAmount())) { requestPayload.setAmount("0"); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java index 0114e0b4..600f571b 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchStatusWorker.java @@ -1,19 +1,23 @@ package org.mifos.processor.bulk.zeebe.worker; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_RATE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Map; import org.mifos.processor.bulk.OperationsAppConfig; import org.mifos.processor.bulk.schema.BatchDTO; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.*; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; import org.springframework.web.client.RestTemplate; import org.springframework.web.util.UriComponentsBuilder; -import java.util.Map; - -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; - -public class BatchStatusWorker extends BaseWorker{ +public class BatchStatusWorker extends BaseWorker { @Autowired public OperationsAppConfig operationsAppConfig; @@ -36,8 +40,8 @@ public void setup() { })); } - private float calculateSuccessPercentage(BatchDTO batchDTO){ - if(batchDTO.getTotal()!=null && batchDTO.getTotal()!=0){ + private float calculateSuccessPercentage(BatchDTO batchDTO) { + if (batchDTO.getTotal() != null && batchDTO.getTotal() != 0) { return (((float) batchDTO.getSuccessful() / batchDTO.getTotal()) * 100); } return 0; @@ -52,8 +56,7 @@ private BatchDTO invokeBatchAggregationApi(String batchId, String tenantId) { UriComponentsBuilder uriBuilder = UriComponentsBuilder.fromUriString(url).queryParam("batchId", batchId); String finalUrl = uriBuilder.toUriString(); - ResponseEntity response = restTemplate.exchange(finalUrl, HttpMethod.GET, - new HttpEntity<>(null, headers), String.class); + ResponseEntity response = restTemplate.exchange(finalUrl, HttpMethod.GET, new HttpEntity<>(null, headers), String.class); String batchAggregationResponse = response != null ? response.getBody() : null; ObjectMapper objectMapper = new ObjectMapper(); BatchDTO batchDTO = null; diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java index 4e59a619..d6a15809 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java @@ -2,15 +2,14 @@ public enum Worker { - PARTY_LOOKUP("partyLookup"), APPROVAL("approval"), ORDERING("ordering"), SPLITTING("splitting"), - FORMATTING("formatting"), BATCH_STATUS("batchStatus"), SEND_CALLBACK("sendCallback"), - MERGE_BACK("mergeSubBatch"), INIT_SUB_BATCH("initSubBatch"), ACCOUNT_LOOKUP("accountLookup"), - ACCOUNT_LOOKUP_CALLBACK("accountLookupCallback"), BATCH_AGGREGATE("batchAggregate"), - AUTHORIZATION("authorization"); + PARTY_LOOKUP("partyLookup"), APPROVAL("approval"), ORDERING("ordering"), SPLITTING("splitting"), FORMATTING("formatting"), BATCH_STATUS( + "batchStatus"), SEND_CALLBACK("sendCallback"), MERGE_BACK("mergeSubBatch"), INIT_SUB_BATCH( + "initSubBatch"), ACCOUNT_LOOKUP("accountLookup"), ACCOUNT_LOOKUP_CALLBACK( + "accountLookupCallback"), BATCH_AGGREGATE("batchAggregate"), AUTHORIZATION("authorization"); private final String value; - private Worker(String s) { + Worker(String s) { value = s; } diff --git a/src/test/java/org/mifos/processor/BulkProcessorApplicationTests.java b/src/test/java/org/mifos/processor/BulkProcessorApplicationTests.java index a9137489..a6473dd9 100644 --- a/src/test/java/org/mifos/processor/BulkProcessorApplicationTests.java +++ b/src/test/java/org/mifos/processor/BulkProcessorApplicationTests.java @@ -2,6 +2,8 @@ import java.util.UUID; import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; @@ -9,9 +11,11 @@ @ActiveProfiles("test") class BulkProcessorApplicationTests { + public Logger logger = LoggerFactory.getLogger(this.getClass()); + @Test void contextLoads() { - System.out.println(UUID.randomUUID()); + logger.debug("{}", UUID.randomUUID()); } } diff --git a/src/test/java/org/mifos/processor/cucumber/stepdef/InitRouteStepDef.java b/src/test/java/org/mifos/processor/cucumber/stepdef/InitRouteStepDef.java index 2c8729af..97717ba6 100644 --- a/src/test/java/org/mifos/processor/cucumber/stepdef/InitRouteStepDef.java +++ b/src/test/java/org/mifos/processor/cucumber/stepdef/InitRouteStepDef.java @@ -48,21 +48,21 @@ public void exchangeVariableBooleanCheck(String variableKey, String variableValu public void callRuntimePayloadTestRoute(String paymentMode) { exchange = template.send("direct:dynamic-payload-setter", exchange -> { exchange.setProperty(PAYMENT_MODE, paymentMode); - exchange.setProperty(TRANSACTION_LIST_ELEMENT, new Transaction() { - { - setId(0); - setRequestId(UUID.randomUUID().toString()); - setPaymentMode(paymentMode); - setAmount("100"); - setPayerIdentifierType("MSISDN"); - setPayeeIdentifierType("MSISDN"); - setPayerIdentifier("1234567890"); - setPayeeIdentifier("0987654321"); - setCurrency("INR"); - } - }); + Transaction transaction = new Transaction(); + transaction.setId(0); + transaction.setRequestId(UUID.randomUUID().toString()); + transaction.setPaymentMode(paymentMode); + transaction.setAmount("100"); + transaction.setPayerIdentifierType("MSISDN"); + transaction.setPayeeIdentifierType("MSISDN"); + transaction.setPayerIdentifier("1234567890"); + transaction.setPayeeIdentifier("0987654321"); + transaction.setCurrency("INR"); + exchange.setProperty(TRANSACTION_LIST_ELEMENT, new Transaction()); + }); + } @And("The body should be of GSMA parcelable") From 28b8ffb1a63073ebe62c8ef2c75bc91443e696b1 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Wed, 8 Nov 2023 17:42:15 +0530 Subject: [PATCH 102/156] GOV-360: Enahnced the splitting logic to support subBatch data creation for individual txn (#126) * Enahnced the splitting logic to support subBatch data creation for individual txn * Fixed the generate sub batch entity logic * Splitting log info * Update init sub batch logic to pass subbatchId in place of bathc id while calling channel api * Fixed the splittign logic * SUB_BATCH_ID variable removed for parent batch * Updated crucial data logging to debug * Checkstyle error fixed --- .../bulk/camel/config/CamelProperties.java | 3 +- .../bulk/camel/routes/InitSubBatchRoute.java | 17 ++-- .../bulk/camel/routes/SplittingRoute.java | 78 ++++++++++++++++++- .../processor/bulk/schema/SubBatchEntity.java | 55 +++++++++++++ .../bulk/zeebe/worker/InitSubBatchWorker.java | 26 +++++++ .../bulk/zeebe/worker/SplittingWorker.java | 9 ++- 6 files changed, 176 insertions(+), 12 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/schema/SubBatchEntity.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 0263c353..581dd6aa 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -19,6 +19,7 @@ private CamelProperties() {} public static final String SUB_BATCH_COUNT = "subBatchCount"; public static final String SUB_BATCH_CREATED = "subBatchCreated"; + public static final String SUB_BATCH_DETAILS = "subBatchDetails"; public static final String SERVER_SUB_BATCH_FILE_NAME_ARRAY = "serverSubBatchFileName"; @@ -78,5 +79,5 @@ private CamelProperties() {} public static final String HEADER_PLATFORM_TENANT_ID = "Platform-TenantId"; public static final String HEADER_CLIENT_CORRELATION_ID = "X-CorrelationID"; public static final String CLIENT_CORRELATION_ID = "clientCorrelationId"; - + public static final String SUB_BATCH_ENTITY = "subBatchEntity"; } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index 49e7c38a..96512210 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -11,6 +11,7 @@ import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYMENT_MODE_TYPE; import static org.mifos.processor.bulk.camel.config.CamelProperties.RESULT_TRANSACTION_LIST; import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_ENTITY; import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_ELEMENT; @@ -27,14 +28,11 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.RESULT_FILE; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SUB_BATCH_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TOTAL_AMOUNT; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.function.Function; import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; @@ -42,6 +40,7 @@ import org.mifos.processor.bulk.config.PaymentModeConfiguration; import org.mifos.processor.bulk.config.PaymentModeMapping; import org.mifos.processor.bulk.config.PaymentModeType; +import org.mifos.processor.bulk.schema.SubBatchEntity; import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.schema.TransactionResult; import org.mifos.processor.bulk.utility.Utils; @@ -85,9 +84,8 @@ public void configure() throws Exception { .process(exchange -> { List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); - Map variables = new HashMap<>(); + Map variables = exchange.getProperty(ZEEBE_VARIABLE, Map.class); variables.put(BATCH_ID, exchange.getProperty(BATCH_ID)); - variables.put(SUB_BATCH_ID, UUID.randomUUID().toString()); variables.put(FILE_NAME, exchange.getProperty(SERVER_FILE_NAME)); variables.put(REQUEST_ID, exchange.getProperty(REQUEST_ID)); variables.put(PURPOSE, exchange.getProperty(PURPOSE)); @@ -190,8 +188,13 @@ public void configure() throws Exception { } }).choice().when(exchangeProperty(EXTERNAL_ENDPOINT_FAILED).isEqualTo(false)) .log(LoggingLevel.DEBUG, "Making API call to endpoint ${exchangeProperty.extEndpoint} and body: ${body}") - .setHeader(Exchange.CONTENT_TYPE, constant("application/json")) - .setHeader(BATCH_ID_HEADER, simple("${exchangeProperty." + BATCH_ID + "}")) + .setHeader(Exchange.CONTENT_TYPE, constant("application/json")).choice() + .when(exchange -> exchange.getProperty(SUB_BATCH_ENTITY, SubBatchEntity.class) != null) + .log("Sub batch entity is not null, hence passing subBatchId while calling channel API").process(exchange -> { + SubBatchEntity subBatchEntity = exchange.getProperty(SUB_BATCH_ENTITY, SubBatchEntity.class); + exchange.getIn().setHeader(BATCH_ID_HEADER, subBatchEntity.getSubBatchId()); + }).otherwise().log("Sub batch entity is null, hence passing batchId while calling channel API") + .setHeader(BATCH_ID_HEADER, simple("${exchangeProperty." + BATCH_ID + "}")).endChoice() .setHeader(HEADER_CLIENT_CORRELATION_ID, simple("${exchangeProperty." + REQUEST_ID + "}")) .setHeader(HEADER_REGISTERING_INSTITUTE_ID, simple("${exchangeProperty." + HEADER_REGISTERING_INSTITUTE_ID + "}")) .process(exchange -> { diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java index ff868ad5..3dce4aec 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java @@ -1,18 +1,32 @@ package org.mifos.processor.bulk.camel.routes; import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; +import static org.mifos.processor.bulk.camel.config.CamelProperties.REGISTERING_INSTITUTE_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_SUB_BATCH_FILE_NAME_ARRAY; import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_COUNT; import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_CREATED; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_DETAILS; import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_FILE_ARRAY; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; +import static org.mifos.processor.bulk.camel.config.CamelProperties.ZEEBE_VARIABLE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CLIENT_CORRELATION_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYER_IDENTIFIER; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_FAILED; import java.io.BufferedReader; import java.io.FileReader; import java.io.FileWriter; import java.util.ArrayList; +import java.util.Date; import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.apache.camel.LoggingLevel; +import org.mifos.processor.bulk.schema.SubBatchEntity; +import org.mifos.processor.bulk.schema.Transaction; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -56,7 +70,7 @@ public void configure() throws Exception { List subBatchFile = new ArrayList<>(); int subBatchCount = 1; for (int i = 0; i < lines.size(); i += subBatchSize) { - String filename = System.currentTimeMillis() + "_" + "sub-batch-" + subBatchCount + ".csv"; + String filename = UUID.randomUUID() + "_" + "sub-batch-" + subBatchCount + ".csv"; FileWriter writer = new FileWriter(filename); writer.write(header); for (int j = i; j < Math.min(i + subBatchSize, lines.size()); j++) { @@ -77,13 +91,71 @@ public void configure() throws Exception { from("direct:upload-sub-batch-file").id("direct:upload-sub-batch-file").log("Starting upload of sub-batch file") .loopDoWhile(exchange -> exchange.getProperty(SUB_BATCH_FILE_ARRAY, List.class).size() > 0).process(exchange -> { List subBatchFile = exchange.getProperty(SUB_BATCH_FILE_ARRAY, List.class); - exchange.setProperty(LOCAL_FILE_PATH, subBatchFile.remove(0)); + String localFilePath = subBatchFile.remove(0); + exchange.setProperty(LOCAL_FILE_PATH, localFilePath); exchange.setProperty(SUB_BATCH_FILE_ARRAY, subBatchFile); - }).to("direct:upload-file").process(exchange -> { + logger.debug("Local file path: {}", localFilePath); + logger.debug("Sub batch file array: {}, ", subBatchFile); + }).log(LoggingLevel.DEBUG, "LOCAL_FILE_PATH: ${exchangeProperty." + LOCAL_FILE_PATH + "}") + .to("direct:generate-sub-batch-entity").log("direct:generate-sub-batch-entity completed").to("direct:upload-file") + .process(exchange -> { String serverFilename = exchange.getProperty(SERVER_FILE_NAME, String.class); List serverSubBatchFile = exchange.getProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, List.class); serverSubBatchFile.add(serverFilename); exchange.setProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, serverSubBatchFile); + logger.debug("Server subbatch filename array: {}", serverSubBatchFile); }); + + // generate subBatchEntityDetails, make sure [LOCAL_FILE_PATH] has the absolute sub batch file path + from("direct:generate-sub-batch-entity").id("direct:generate-sub-batch-entity").log("Generating sub batch entity") + .to("direct:get-transaction-array").process(exchange -> { + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + Map zeebeVariables = exchange.getProperty(ZEEBE_VARIABLE, Map.class); + String serverFileName = exchange.getProperty(LOCAL_FILE_PATH, String.class); + + logger.info("Generating sub batch entity for file {}", serverFileName); + if (transactionList.isEmpty()) { + logger.info("Transaction list is empty"); + return; + } + + Long totalAmount = getTotalAmount(transactionList); + + SubBatchEntity subBatchEntity = getDefaultSubBatchEntity(); + subBatchEntity.setBatchId((String) zeebeVariables.get(BATCH_ID)); + subBatchEntity.setSubBatchId(UUID.randomUUID().toString()); + subBatchEntity.setRequestId((String) zeebeVariables.get(REQUEST_ID)); + subBatchEntity.setCorrelationId((String) zeebeVariables.get(CLIENT_CORRELATION_ID)); + subBatchEntity.setPayerFsp((String) zeebeVariables.get(PAYER_IDENTIFIER)); + subBatchEntity.setRegisteringInstitutionId((String) zeebeVariables.get(REGISTERING_INSTITUTE_ID)); + subBatchEntity.setPaymentMode(transactionList.get(0).getPaymentMode()); + subBatchEntity.setRequestFile(serverFileName); + subBatchEntity.setTotalTransactions((long) transactionList.size()); + subBatchEntity.setOngoing((long) transactionList.size()); + subBatchEntity.setTotalAmount(totalAmount); + subBatchEntity.setOngoingAmount(totalAmount); + subBatchEntity.setStartedAt(new Date(System.currentTimeMillis())); + + logger.debug("SubBatchEntity: {}", objectMapper.writeValueAsString(subBatchEntity)); + // update the sub batch details array + List subBatchEntityList = exchange.getProperty(SUB_BATCH_DETAILS, List.class); + subBatchEntityList.add(subBatchEntity); + exchange.setProperty(SUB_BATCH_DETAILS, subBatchEntityList); + logger.debug("generate-sub-batch-entity route end: {}", objectMapper.writeValueAsString(subBatchEntityList)); + }); + } + + private SubBatchEntity getDefaultSubBatchEntity() { + SubBatchEntity subBatchEntity = new SubBatchEntity(); + subBatchEntity.setAllEmptyAmount(); + return subBatchEntity; + } + + private long getTotalAmount(List transactionList) { + long totalAmount = 0L; + for (Transaction transaction : transactionList) { + totalAmount += Long.parseLong(transaction.getAmount()); + } + return totalAmount; } } diff --git a/src/main/java/org/mifos/processor/bulk/schema/SubBatchEntity.java b/src/main/java/org/mifos/processor/bulk/schema/SubBatchEntity.java new file mode 100644 index 00000000..5aa98f41 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/SubBatchEntity.java @@ -0,0 +1,55 @@ +package org.mifos.processor.bulk.schema; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import java.util.Date; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +public class SubBatchEntity { + + private String batchId; + private String subBatchId; + private String requestId; + private String requestFile; + private String resultFile; + private String note; + private String paymentMode; + private String registeringInstitutionId; + private String payerFsp; + private String correlationId; + + private Long totalTransactions; + private Long ongoing; + private Long failed; + private Long completed; + private Long totalAmount; + private Long ongoingAmount; + private Long failedAmount; + private Long completedAmount; + private Long workflowKey; + private Long workflowInstanceKey; + private Long approvedAmount; + private Long approvedCount; + + private Date resultGeneratedAt; + private Date startedAt; + private Date completedAt; + + @JsonIgnore + public void setAllEmptyAmount() { + setTotalTransactions(0L); + setOngoing(0L); + setFailed(0L); + setCompleted(0L); + setTotalAmount(0L); + setOngoingAmount(0L); + setFailedAmount(0L); + setCompletedAmount(0L); + setApprovedAmount(0L); + setApprovedCount(0L); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java index d206d445..2acacf11 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java @@ -1,7 +1,10 @@ package org.mifos.processor.bulk.zeebe.worker; import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_DETAILS; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_ENTITY; import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.ZEEBE_VARIABLE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_FAILURE_SUB_BATCHES; @@ -14,17 +17,24 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SUB_BATCHES; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.ArrayList; import java.util.List; import java.util.Map; import org.apache.camel.Exchange; import org.apache.camel.support.DefaultExchange; import org.mifos.processor.bulk.camel.routes.RouteId; +import org.mifos.processor.bulk.schema.SubBatchEntity; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @Component public class InitSubBatchWorker extends BaseWorker { + @Autowired + private ObjectMapper objectMapper; + @Override public void setup() { @@ -55,7 +65,21 @@ public void setup() { subBatches.add((String) variables.get(FILE_NAME)); } + List subBatchObjectList = (List) variables.get(SUB_BATCH_DETAILS); + logger.debug("Subbatch entity list in init sub batch worker: {}", subBatchObjectList); + + List subBatchEntityList = objectMapper.convertValue(subBatchObjectList, new TypeReference<>() {}); + String fileName = subBatches.remove(0); + SubBatchEntity subBatchEntity = null; + + for (SubBatchEntity subBatch : subBatchEntityList) { + if (subBatch.getRequestFile().contains(fileName)) { + subBatchEntity = subBatch; + logger.info("SubBatchEntity found"); + } + } + logger.debug("BatchEntity for this subbatch is {}", objectMapper.writeValueAsString(subBatchEntity)); Exchange exchange = new DefaultExchange(camelContext); exchange.setProperty(TENANT_NAME, variables.get(TENANT_ID)); @@ -63,6 +87,8 @@ public void setup() { exchange.setProperty(BATCH_ID, variables.get(BATCH_ID)); exchange.setProperty(REQUEST_ID, variables.get(REQUEST_ID)); exchange.setProperty(PURPOSE, variables.get(PURPOSE)); + exchange.setProperty(ZEEBE_VARIABLE, variables); + exchange.setProperty(SUB_BATCH_ENTITY, subBatchEntity); sendToCamelRoute(RouteId.INIT_SUB_BATCH, exchange); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java index fb0d8c85..50a52251 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java @@ -3,6 +3,8 @@ import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_SUB_BATCH_FILE_NAME_ARRAY; import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_CREATED; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_DETAILS; +import static org.mifos.processor.bulk.camel.config.CamelProperties.ZEEBE_VARIABLE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_FAILURE_SUB_BATCHES; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_SUCCESS_SUB_BATCHES; @@ -15,6 +17,7 @@ import org.apache.camel.Exchange; import org.apache.camel.support.DefaultExchange; import org.mifos.processor.bulk.camel.routes.RouteId; +import org.mifos.processor.bulk.schema.SubBatchEntity; import org.springframework.stereotype.Component; @Component @@ -29,7 +32,7 @@ public void setup() { * zeebeVariable [SPLITTING_FAILED, SUB_BATCHES, SUB_BATCH_CREATED] */ newWorker(Worker.SPLITTING, (client, job) -> { - logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); + logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); Map variables = job.getVariablesAsMap(); if (workerConfig.isSplittingWorkerEnabled) { variables.put(SPLITTING_FAILED, false); @@ -38,6 +41,8 @@ public void setup() { String filename = (String) variables.get(FILE_NAME); Exchange exchange = new DefaultExchange(camelContext); exchange.setProperty(SERVER_FILE_NAME, filename); + exchange.setProperty(ZEEBE_VARIABLE, variables); + exchange.setProperty(SUB_BATCH_DETAILS, new ArrayList()); try { sendToCamelRoute(RouteId.SPLITTING, exchange); @@ -56,11 +61,13 @@ public void setup() { variables.put(SPLITTING_FAILED, false); variables.put(SUB_BATCHES, serverSubBatchFileList); + variables.put(SUB_BATCH_DETAILS, exchange.getProperty(SUB_BATCH_DETAILS, ArrayList.class)); variables.put(INIT_SUCCESS_SUB_BATCHES, new ArrayList()); variables.put(INIT_FAILURE_SUB_BATCHES, new ArrayList()); variables.put(SUB_BATCH_CREATED, subBatchCreated); client.newCompleteCommand(job.getKey()).variables(variables).send(); + logger.info("Splitting worker completed"); }); } From 6afd2b47404a122258efc237c0954de03dda8cae Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:15:50 +0530 Subject: [PATCH 103/156] GOV-389 Subscribe to a eventType logic added (#81) * GOV-389 Subscribe to a eventType logic added * Builder renamed * Updated the org name in SS config * Fixed the checkstyle build failure --- .../bulk/camel/config/CamelProperties.java | 1 + .../bulk/camel/routes/PubSubRoute.java | 54 ++++++++ .../processor/bulk/config/PubSubConfig.java | 18 +++ .../bulk/config/SecurityServerConfig.java | 32 +++++ .../bulk/schema/SubscriptionDTO.java | 123 ++++++++++++++++++ .../bulk/schema/SubscriptionOptionsDTO.java | 50 +++++++ .../bulk/service/SubscriptionService.java | 10 ++ .../bulk/service/SubscriptionServiceImpl.java | 34 +++++ src/main/resources/application.yaml | 19 +++ 9 files changed, 341 insertions(+) create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/PubSubRoute.java create mode 100644 src/main/java/org/mifos/processor/bulk/config/PubSubConfig.java create mode 100644 src/main/java/org/mifos/processor/bulk/config/SecurityServerConfig.java create mode 100644 src/main/java/org/mifos/processor/bulk/schema/SubscriptionDTO.java create mode 100644 src/main/java/org/mifos/processor/bulk/schema/SubscriptionOptionsDTO.java create mode 100644 src/main/java/org/mifos/processor/bulk/service/SubscriptionService.java create mode 100644 src/main/java/org/mifos/processor/bulk/service/SubscriptionServiceImpl.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 581dd6aa..898bb873 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -80,4 +80,5 @@ private CamelProperties() {} public static final String HEADER_CLIENT_CORRELATION_ID = "X-CorrelationID"; public static final String CLIENT_CORRELATION_ID = "clientCorrelationId"; public static final String SUB_BATCH_ENTITY = "subBatchEntity"; + public static final String EVENT_TYPE = "eventType"; } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/PubSubRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/PubSubRoute.java new file mode 100644 index 00000000..c108c360 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/PubSubRoute.java @@ -0,0 +1,54 @@ +package org.mifos.processor.bulk.camel.routes; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.ENDPOINT; +import static org.mifos.processor.bulk.camel.config.CamelProperties.EVENT_TYPE; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HOST; + +import org.apache.camel.Exchange; +import org.mifos.processor.bulk.config.PubSubConfig; +import org.mifos.processor.bulk.config.SecurityServerConfig; +import org.mifos.processor.bulk.schema.SubscriptionDTO; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class PubSubRoute extends BaseRouteBuilder { + + private final PubSubConfig pubSubConfig; + private final SecurityServerConfig securityServerConfig; + + @Value("${gov-stack-client.header-key}") + private String govStackClientHeaderKey; + + @Value("${gov-stack-client.header-value}") + private String govStackClientHeaderValue; + + public PubSubRoute(PubSubConfig pubSubConfig, SecurityServerConfig securityServerConfig) { + this.pubSubConfig = pubSubConfig; + this.securityServerConfig = securityServerConfig; + } + + @Override + public void configure() throws Exception { + + // needs EVENT_TYPE input from exchange + from("direct:subscribe").id("direct:subscribe") + .setBody(exchange -> getEventTypeSpecificSubscriptionDTO(exchange.getProperty(EVENT_TYPE, String.class))) + .setHeader(Exchange.HTTP_METHOD, constant(HttpRequestMethod.POST.toString())) + .setHeader(govStackClientHeaderKey, constant(govStackClientHeaderValue)) + .setProperty(HOST, constant(securityServerConfig.host)).setProperty(ENDPOINT, constant(securityServerConfig.subscribingUrl)) + .to("direct:external-api-calling"); + + } + + private SubscriptionDTO getDefaultSubscriptionDTO() { + return SubscriptionDTO.subscriptionDTOBuilder.roomCode(pubSubConfig.roomCode).roomClass(pubSubConfig.roomClass) + .srcOperationId("bulkProcessing").srcServiceCode("bulk").dstOperationId("newRecord").dstServiceCode("bulk").build(); + } + + private SubscriptionDTO getEventTypeSpecificSubscriptionDTO(String eventType) { + SubscriptionDTO subscriptionDTO = getDefaultSubscriptionDTO(); + subscriptionDTO.setDstServiceCode(eventType); // todo update once confirmed which field is for eventType + return subscriptionDTO; + } +} diff --git a/src/main/java/org/mifos/processor/bulk/config/PubSubConfig.java b/src/main/java/org/mifos/processor/bulk/config/PubSubConfig.java new file mode 100644 index 00000000..23d39064 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/config/PubSubConfig.java @@ -0,0 +1,18 @@ +package org.mifos.processor.bulk.config; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class PubSubConfig { + + @Value("${pubsub.room.code}") + public String roomCode; + + @Value("${pubsub.room.class}") + public String roomClass; + + @Value("${pubsub.event.type}") + public String eventType; + +} diff --git a/src/main/java/org/mifos/processor/bulk/config/SecurityServerConfig.java b/src/main/java/org/mifos/processor/bulk/config/SecurityServerConfig.java new file mode 100644 index 00000000..fd27c6a1 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/config/SecurityServerConfig.java @@ -0,0 +1,32 @@ +package org.mifos.processor.bulk.config; + +import javax.annotation.PostConstruct; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class SecurityServerConfig { + + @Value("${security-server.host}") + public String host; + + @Value("${security-server.baseuri}") + public String baseUri; + + @Value("${security-server.country}") + public String country; + + @Value("${security-server.organisation}") + public String organisation; + + @Value("${security-server.endpoints.subs}") + public String subscribingEndpoint; + + public String subscribingUrl; + + @PostConstruct + public void setup() { + subscribingUrl = host + baseUri + subscribingEndpoint; + subscribingUrl = subscribingUrl.replace("{country}", country).replace("{orgs}", organisation); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/schema/SubscriptionDTO.java b/src/main/java/org/mifos/processor/bulk/schema/SubscriptionDTO.java new file mode 100644 index 00000000..645fd957 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/SubscriptionDTO.java @@ -0,0 +1,123 @@ +package org.mifos.processor.bulk.schema; + +import java.util.UUID; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class SubscriptionDTO { + + private String id; + + private String roomClass; + + private String roomCode; + + private String srcServiceCode; + + private String srcOperationId; + + private String dstServiceCode; + + private String dstOperationId; + + private String delivery; + + private SubscriptionOptionsDTO options; + + public static SubscriptionDTOBuilder subscriptionDTOBuilder = new SubscriptionDTOBuilder(); + + public static class SubscriptionDTOBuilder { + + private String id; + private String roomClass; + private String roomCode; + private String srcServiceCode; + private String srcOperationId; + private String dstServiceCode; + private String dstOperationId; + private String delivery; + private SubscriptionOptionsDTO options; + + public SubscriptionDTOBuilder roomClass(String roomClass) { + this.roomClass = roomClass; + return this; + } + + public SubscriptionDTOBuilder roomCode(String roomCode) { + this.roomCode = roomCode; + return this; + } + + public SubscriptionDTOBuilder srcServiceCode(String srcServiceCode) { + this.srcServiceCode = srcServiceCode; + return this; + } + + public SubscriptionDTOBuilder srcOperationId(String srcOperationId) { + this.srcOperationId = srcOperationId; + return this; + } + + public SubscriptionDTOBuilder dstServiceCode(String dstServiceCode) { + this.dstServiceCode = dstServiceCode; + return this; + } + + public SubscriptionDTOBuilder dstOperationId(String dstOperationId) { + this.dstOperationId = dstOperationId; + return this; + } + + public SubscriptionDTOBuilder delivery(String delivery) { + this.delivery = delivery; + return this; + } + + public SubscriptionDTOBuilder options(SubscriptionOptionsDTO options) { + this.options = options; + return this; + } + + private void check() { + if (this.id == null) { + this.id = UUID.randomUUID().toString(); + } + if (this.options == null) { + this.options = SubscriptionOptionsDTO.subscriptionOptionsDTOBuilder.build(); + } + if (this.delivery == null) { + this.delivery = "PUSH"; + } + if (roomClass == null) { + throw new RuntimeException("roomClass field cant be null"); + } + if (roomCode == null) { + throw new RuntimeException("roomCode field cant be null"); + } + if (srcServiceCode == null) { + throw new RuntimeException("srcServiceCode field cant be null"); + } + if (srcOperationId == null) { + throw new RuntimeException("srcOperationId field cant be null"); + } + if (dstServiceCode == null) { + throw new RuntimeException("dstServiceCode field cant be null"); + } + if (dstOperationId == null) { + throw new RuntimeException("dstOperationId field cant be null"); + } + } + + public SubscriptionDTO build() { + check(); + return new SubscriptionDTO(this.id, this.roomClass, this.roomCode, this.srcServiceCode, this.srcOperationId, + this.dstServiceCode, this.dstOperationId, this.delivery, this.options); + } + } +} diff --git a/src/main/java/org/mifos/processor/bulk/schema/SubscriptionOptionsDTO.java b/src/main/java/org/mifos/processor/bulk/schema/SubscriptionOptionsDTO.java new file mode 100644 index 00000000..9cb1858b --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/SubscriptionOptionsDTO.java @@ -0,0 +1,50 @@ +package org.mifos.processor.bulk.schema; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class SubscriptionOptionsDTO { + + @JsonProperty("TTL") + private int ttl; + + @JsonProperty("FIFO") + private boolean fifo; + + public static SubscriptionOptionsDTOBuilder subscriptionOptionsDTOBuilder = new SubscriptionOptionsDTOBuilder(); + + public static class SubscriptionOptionsDTOBuilder { + + private Integer ttl; + + private Boolean fifo; + + public SubscriptionOptionsDTOBuilder tTL(int tTL) { + this.ttl = tTL; + return this; + } + + public SubscriptionOptionsDTOBuilder fIFO(boolean fIFO) { + this.fifo = fIFO; + return this; + } + + public SubscriptionOptionsDTO build() { + if (this.ttl == null) { + this.ttl = 3600; + } + if (this.fifo == null) { + this.fifo = false; + } + return new SubscriptionOptionsDTO(this.ttl, this.fifo); + } + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/service/SubscriptionService.java b/src/main/java/org/mifos/processor/bulk/service/SubscriptionService.java new file mode 100644 index 00000000..6c067201 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/service/SubscriptionService.java @@ -0,0 +1,10 @@ +package org.mifos.processor.bulk.service; + +import org.springframework.stereotype.Service; + +@Service +public interface SubscriptionService { + + void subscribeToEvent(); + +} diff --git a/src/main/java/org/mifos/processor/bulk/service/SubscriptionServiceImpl.java b/src/main/java/org/mifos/processor/bulk/service/SubscriptionServiceImpl.java new file mode 100644 index 00000000..774a0f60 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/service/SubscriptionServiceImpl.java @@ -0,0 +1,34 @@ +package org.mifos.processor.bulk.service; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.EVENT_TYPE; + +import javax.annotation.PostConstruct; +import org.apache.camel.CamelContext; +import org.apache.camel.Exchange; +import org.apache.camel.ProducerTemplate; +import org.apache.camel.support.DefaultExchange; +import org.springframework.stereotype.Service; + +@Service +public class SubscriptionServiceImpl implements SubscriptionService { + + private final ProducerTemplate producerTemplate; + + protected final CamelContext camelContext; + + public String eventType; + + public SubscriptionServiceImpl(ProducerTemplate producerTemplate, CamelContext camelContext) { + this.producerTemplate = producerTemplate; + this.camelContext = camelContext; + } + + @PostConstruct + @Override + public void subscribeToEvent() { + Exchange exchange = new DefaultExchange(camelContext); + exchange.setProperty(EVENT_TYPE, eventType); + producerTemplate.send("direct:subscribe", exchange); + } + +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index f4fc2a29..879f43bf 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -194,3 +194,22 @@ payment-mode: debulkingDfspid: "lion" batch-authorization: callback-url: "https://ph-ee-connector-bulk:80/authorization/callback" + +pubsub: + room: + code: "covid-19" + class: "GOV" + event: + type: "bulk" + +security-server: + country: "INDIA" + organisation: mifos + host: https://SECURITYSERVER + baseuri: /r1/{country}/GOV/{orgs} + endpoints: + subs: /room/subs + +gov-stack-client: + header-key: "X-GovStack-Client" + header-value: "PAYMENT-BB" From 274a5d416521702f461943afffa28e8da43d82c0 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Tue, 21 Nov 2023 14:37:53 +0530 Subject: [PATCH 104/156] GOV:325 Implementation of de duplication story (#128) * Added de duplication worker, no implementation * Implemented the de duplication worker * Removed de dup logic from ordering worker * Removed the unnecesary comments --- .../bulk/camel/config/CamelProperties.java | 2 + .../bulk/camel/routes/DeDuplicationRoute.java | 119 ++++++++++++++++++ .../camel/routes/ProcessorStartRoute.java | 2 + .../processor/bulk/camel/routes/RouteId.java | 3 +- .../processor/bulk/zeebe/ZeebeVariables.java | 7 ++ .../zeebe/worker/DeDuplicationWorker.java | 55 ++++++++ .../bulk/zeebe/worker/OrderingWorker.java | 7 -- .../processor/bulk/zeebe/worker/Worker.java | 6 +- 8 files changed, 190 insertions(+), 11 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/DeDuplicationRoute.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/DeDuplicationWorker.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 898bb873..e4c71cd0 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -81,4 +81,6 @@ private CamelProperties() {} public static final String CLIENT_CORRELATION_ID = "clientCorrelationId"; public static final String SUB_BATCH_ENTITY = "subBatchEntity"; public static final String EVENT_TYPE = "eventType"; + public static final String DUPLICATE_TRANSACTION_LIST = "duplicateTransactionList"; + public static final String ORIGINAL_TRANSACTION_LIST = "originalTransactionList"; } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/DeDuplicationRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/DeDuplicationRoute.java new file mode 100644 index 00000000..53619354 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/DeDuplicationRoute.java @@ -0,0 +1,119 @@ +package org.mifos.processor.bulk.camel.routes; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.DUPLICATE_TRANSACTION_LIST; +import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; +import static org.mifos.processor.bulk.camel.config.CamelProperties.ORIGINAL_TRANSACTION_LIST; +import static org.mifos.processor.bulk.camel.config.CamelProperties.OVERRIDE_HEADER; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DE_DUPLICATION_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DUPLICATE_TRANSACTION_COUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DUPLICATE_TRANSACTION_FILE; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.mifos.processor.bulk.schema.Transaction; +import org.springframework.stereotype.Component; + +@Component +public class DeDuplicationRoute extends BaseRouteBuilder { + + @Override + public void configure() throws Exception { + from(RouteId.DE_DUPLICATION.getValue()).id(RouteId.DE_DUPLICATION.getValue()) + .log("Started route " + RouteId.DE_DUPLICATION.getValue()).to("direct:download-file").to("direct:get-transaction-array") + .process(exchange -> { + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + + if (Objects.isNull(transactionList) || transactionList.isEmpty()) { + exchange.setProperty(DE_DUPLICATION_FAILED, false); + exchange.setProperty(DUPLICATE_TRANSACTION_COUNT, 0); + } + + int duplicateTxnCount = 0; + List duplicateTransactionList = new ArrayList<>(); // contains the duplicate + // transaction + List originalTransactionList = new ArrayList<>(); // contains the original txn after + // removing duplicate + Set set = new HashSet<>(); + + for (Transaction transaction : transactionList) { + String payeeDetail = fetchPayeeDetail(transaction); + if (set.contains(payeeDetail)) { + transaction.setNote("Duplicate transaction."); + duplicateTransactionList.add(transaction); + duplicateTxnCount++; + } else { + set.add(payeeDetail); + originalTransactionList.add(transaction); + } + } + + log.info("Duplicate txn: {} and count: {}", duplicateTransactionList, duplicateTxnCount); + + exchange.setProperty(DUPLICATE_TRANSACTION_COUNT, duplicateTxnCount); + exchange.setProperty(DUPLICATE_TRANSACTION_LIST, duplicateTransactionList); + exchange.setProperty(ORIGINAL_TRANSACTION_LIST, originalTransactionList); + }).choice().when(exchange -> exchange.getProperty(DUPLICATE_TRANSACTION_COUNT, Integer.class) > 0) + .log("Updating original transaction list") + .setProperty(TRANSACTION_LIST, simple("${exchangeProperty." + ORIGINAL_TRANSACTION_LIST + "}")) + .setProperty(LOCAL_FILE_PATH, simple("${exchangeProperty." + SERVER_FILE_NAME + "}")) + .setProperty(OVERRIDE_HEADER, constant(true)).to("direct:update-file").to("direct:upload-file").process(exchange -> { + String originalFileServerName = exchange.getProperty(SERVER_FILE_NAME, String.class); + String duplicateFileName = "duplicate_transaction_" + originalFileServerName; + + exchange.setProperty(DUPLICATE_TRANSACTION_FILE, duplicateFileName); + }).log("Updating duplicate transaction list") + .setProperty(TRANSACTION_LIST, simple("${exchangeProperty." + DUPLICATE_TRANSACTION_LIST + "}")) + .setProperty(LOCAL_FILE_PATH, simple("${exchangeProperty." + DUPLICATE_TRANSACTION_FILE + "}")) + .setProperty(OVERRIDE_HEADER, constant(true)).to("direct:update-file").to("direct:upload-file").process(exchange -> { + // checking if file upload was success or + String serverFileName = exchange.getProperty(SERVER_FILE_NAME, String.class); + if (serverFileName == null) { + exchange.setProperty(DE_DUPLICATION_FAILED, true); + } else { + exchange.setProperty(DE_DUPLICATION_FAILED, false); + } + }).otherwise().log("No duplicate transaction found").setProperty(DE_DUPLICATION_FAILED, constant(false)).endChoice(); + } + + private void removeDuplicatesIfOrderingDisabled(List transactionList) { + Set set = new HashSet<>(); + + if (Objects.isNull(transactionList)) { + return; + } + + for (Transaction transaction : transactionList) { + String payeeDetail = fetchPayeeDetail(transaction); + if (set.contains(payeeDetail)) { + transaction.setNote("Duplicate transaction."); + } else { + set.add(payeeDetail); + } + } + } + + private Map getTransactionPayeeDetailHashMap(List transactionList) { + Map payeeDetailTransactionMap = new HashMap<>(); + for (Transaction transaction : transactionList) { + payeeDetailTransactionMap.put(fetchPayeeDetail(transaction), transaction); + } + return payeeDetailTransactionMap; + } + + private String fetchPayeeDetail(Transaction transaction) { + String payeeIdentifier = transaction.getPayeeIdentifier(); + String payeeIdentifierType = transaction.getPayeeIdentifierType(); + String amount = transaction.getAmount(); + String currency = transaction.getCurrency(); + + return String.format("%s%s%s%s", payeeIdentifier, payeeIdentifierType, amount, currency); + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 5b7ebda5..374ddc67 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -23,6 +23,7 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CLIENT_CORRELATION_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_THRESHOLD; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_THRESHOLD_CHECK_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DE_DUPLICATION_ENABLE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FORMATTING_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.IS_FILE_VALID; @@ -471,6 +472,7 @@ private Map setConfigProperties(Map variables) { variables.put(PARTY_LOOKUP_ENABLED, workerConfig.isPartyLookUpWorkerEnabled); variables.put(AUTHORIZATION_ENABLED, workerConfig.isAuthorizationWorkerEnabled); variables.put(APPROVAL_ENABLED, workerConfig.isApprovalWorkerEnabled); + variables.put(DE_DUPLICATION_ENABLE, workerConfig.isTransactionDeduplicationEnabled); variables.put(ORDERING_ENABLED, workerConfig.isOrderingWorkerEnabled); variables.put(SPLITTING_ENABLED, workerConfig.isSplittingWorkerEnabled); variables.put(FORMATTING_ENABLED, workerConfig.isFormattingWorkerEnabled); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java b/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java index 82309c7d..1fae2316 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java @@ -4,7 +4,8 @@ public enum RouteId { PARTY_LOOKUP("direct:partyLookup"), APPROVAL("direct:approval"), ORDERING("direct:ordering"), SPLITTING("direct:splitting"), FORMATTING( "direct:formatting"), BATCH_STATUS("direct:batchStatus"), SEND_CALLBACK("direct:sendCallback"), MERGE_BACK( - "direct:mergeSubBatch"), INIT_SUB_BATCH("direct:init-sub-batches"), BATCH_AGGREGATE("direct:batch-aggregate"); + "direct:mergeSubBatch"), INIT_SUB_BATCH( + "direct:init-sub-batches"), BATCH_AGGREGATE("direct:batch-aggregate"), DE_DUPLICATION("direct:deDuplication"); private final String value; diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 8a1d3e94..86c56051 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -27,6 +27,7 @@ private ZeebeVariables() {} public static final String PARTY_LOOKUP_FAILED = "partyLookupFailed"; public static final String APPROVAL_FAILED = "approvalFailed"; + public static final String DE_DUPLICATION_FAILED = "deduplicationFailed"; public static final String ORDERING_FAILED = "orderingFailed"; public static final String SPLITTING_FAILED = "splittingFailed"; public static final String FORMATTING_FAILED = "formattingFailed"; @@ -48,6 +49,8 @@ private ZeebeVariables() {} public static final String APPROVAL_ENABLED = "approvalEnabled"; + public static final String DE_DUPLICATION_ENABLE = "deduplicationEnabled"; + public static final String ORDERING_ENABLED = "orderingEnabled"; public static final String SPLITTING_ENABLED = "splittingEnabled"; @@ -157,4 +160,8 @@ private ZeebeVariables() {} public static final String AUTHORIZATION_RESPONSE = "authorizationResponse"; + public static final String DUPLICATE_TRANSACTION_FILE = "duplicateTransactionFile"; + + public static final String DUPLICATE_TRANSACTION_COUNT = "duplicateTransactionCount"; + } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/DeDuplicationWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/DeDuplicationWorker.java new file mode 100644 index 00000000..02bece35 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/DeDuplicationWorker.java @@ -0,0 +1,55 @@ +package org.mifos.processor.bulk.zeebe.worker; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DE_DUPLICATION_ENABLE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DE_DUPLICATION_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DUPLICATE_TRANSACTION_COUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DUPLICATE_TRANSACTION_FILE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; + +import java.util.Map; +import org.apache.camel.Exchange; +import org.apache.camel.support.DefaultExchange; +import org.mifos.processor.bulk.camel.routes.RouteId; +import org.springframework.stereotype.Component; + +@Component +public class DeDuplicationWorker extends BaseWorker { + + @Override + public void setup() { + newWorker(Worker.DE_DEPLICATION, (client, job) -> { + logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); + logger.info("Started {} worker", Worker.DE_DEPLICATION.getValue()); + Map variables = job.getVariablesAsMap(); + + if ((Boolean) variables.get(DE_DUPLICATION_ENABLE)) { + variables.put(DE_DUPLICATION_FAILED, false); + } + + Exchange exchange = new DefaultExchange(camelContext); + + String filename = (String) variables.get(FILE_NAME); + + logger.info("Filename in worker before duplication is: {}", filename); + + exchange.setProperty(SERVER_FILE_NAME, filename); + + sendToCamelRoute(RouteId.DE_DUPLICATION, exchange); + + boolean deDuplicationFailed = (Boolean) exchange.getProperty(DE_DUPLICATION_FAILED); + int duplicateTransactionCount = exchange.getProperty(DUPLICATE_TRANSACTION_COUNT, Integer.class); + if (duplicateTransactionCount > 0) { + // if duplicate txn exist + variables.put(DUPLICATE_TRANSACTION_FILE, exchange.getProperty(DUPLICATE_TRANSACTION_FILE, String.class)); + } + variables.put(DE_DUPLICATION_FAILED, deDuplicationFailed); + variables.put(DUPLICATE_TRANSACTION_COUNT, duplicateTransactionCount); + + logger.debug("Zeebe variables in dedup: {}", variables); + client.newCompleteCommand(job.getKey()).variables(variables).send(); + logger.info("Completed {} worker", Worker.DE_DEPLICATION); + }); + + } +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java index 83e41af8..d8970e77 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/OrderingWorker.java @@ -1,7 +1,6 @@ package org.mifos.processor.bulk.zeebe.worker; import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; -import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERED_BY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERING_FAILED; @@ -47,12 +46,6 @@ public void setup() { variables.put(ORDERING_FAILED, false); variables.put(ORDERED_BY, exchange.getProperty(ORDERED_BY)); } - - if (workerConfig.isTransactionDeduplicationEnabled) { - List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); - removeDuplicates(transactionList, workerConfig.isOrderingWorkerEnabled); - variables.put(TRANSACTION_LIST, transactionList); - } client.newCompleteCommand(job.getKey()).variables(variables).send(); }); } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java index d6a15809..49610a69 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java @@ -3,9 +3,9 @@ public enum Worker { PARTY_LOOKUP("partyLookup"), APPROVAL("approval"), ORDERING("ordering"), SPLITTING("splitting"), FORMATTING("formatting"), BATCH_STATUS( - "batchStatus"), SEND_CALLBACK("sendCallback"), MERGE_BACK("mergeSubBatch"), INIT_SUB_BATCH( - "initSubBatch"), ACCOUNT_LOOKUP("accountLookup"), ACCOUNT_LOOKUP_CALLBACK( - "accountLookupCallback"), BATCH_AGGREGATE("batchAggregate"), AUTHORIZATION("authorization"); + "batchStatus"), SEND_CALLBACK("sendCallback"), MERGE_BACK("mergeSubBatch"), INIT_SUB_BATCH("initSubBatch"), ACCOUNT_LOOKUP( + "accountLookup"), ACCOUNT_LOOKUP_CALLBACK("accountLookupCallback"), BATCH_AGGREGATE( + "batchAggregate"), AUTHORIZATION("authorization"), DE_DEPLICATION("deduplicate"); private final String value; From 64eec863571d8502b224bc92d188f738f324fef8 Mon Sep 17 00:00:00 2001 From: Danish Jamal <31315800+danishjamal104@users.noreply.github.com> Date: Wed, 22 Nov 2023 12:52:40 +0530 Subject: [PATCH 105/156] Renamed the zeebe variable duplicatxnfile to failedTxnFile (#129) --- .../processor/bulk/camel/routes/DeDuplicationRoute.java | 6 +++--- .../java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java | 2 +- .../processor/bulk/zeebe/worker/DeDuplicationWorker.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/DeDuplicationRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/DeDuplicationRoute.java index 53619354..032fe09d 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/DeDuplicationRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/DeDuplicationRoute.java @@ -8,7 +8,7 @@ import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DE_DUPLICATION_FAILED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DUPLICATE_TRANSACTION_COUNT; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DUPLICATE_TRANSACTION_FILE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FAILED_TRANSACTION_FILE; import java.util.ArrayList; import java.util.HashMap; @@ -67,10 +67,10 @@ public void configure() throws Exception { String originalFileServerName = exchange.getProperty(SERVER_FILE_NAME, String.class); String duplicateFileName = "duplicate_transaction_" + originalFileServerName; - exchange.setProperty(DUPLICATE_TRANSACTION_FILE, duplicateFileName); + exchange.setProperty(FAILED_TRANSACTION_FILE, duplicateFileName); }).log("Updating duplicate transaction list") .setProperty(TRANSACTION_LIST, simple("${exchangeProperty." + DUPLICATE_TRANSACTION_LIST + "}")) - .setProperty(LOCAL_FILE_PATH, simple("${exchangeProperty." + DUPLICATE_TRANSACTION_FILE + "}")) + .setProperty(LOCAL_FILE_PATH, simple("${exchangeProperty." + FAILED_TRANSACTION_FILE + "}")) .setProperty(OVERRIDE_HEADER, constant(true)).to("direct:update-file").to("direct:upload-file").process(exchange -> { // checking if file upload was success or String serverFileName = exchange.getProperty(SERVER_FILE_NAME, String.class); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 86c56051..b08d94ee 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -160,7 +160,7 @@ private ZeebeVariables() {} public static final String AUTHORIZATION_RESPONSE = "authorizationResponse"; - public static final String DUPLICATE_TRANSACTION_FILE = "duplicateTransactionFile"; + public static final String FAILED_TRANSACTION_FILE = "failedTransactionFile"; public static final String DUPLICATE_TRANSACTION_COUNT = "duplicateTransactionCount"; diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/DeDuplicationWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/DeDuplicationWorker.java index 02bece35..5448dcdd 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/DeDuplicationWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/DeDuplicationWorker.java @@ -4,7 +4,7 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DE_DUPLICATION_ENABLE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DE_DUPLICATION_FAILED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DUPLICATE_TRANSACTION_COUNT; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DUPLICATE_TRANSACTION_FILE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FAILED_TRANSACTION_FILE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; import java.util.Map; @@ -41,7 +41,7 @@ public void setup() { int duplicateTransactionCount = exchange.getProperty(DUPLICATE_TRANSACTION_COUNT, Integer.class); if (duplicateTransactionCount > 0) { // if duplicate txn exist - variables.put(DUPLICATE_TRANSACTION_FILE, exchange.getProperty(DUPLICATE_TRANSACTION_FILE, String.class)); + variables.put(FAILED_TRANSACTION_FILE, exchange.getProperty(FAILED_TRANSACTION_FILE, String.class)); } variables.put(DE_DUPLICATION_FAILED, deDuplicationFailed); variables.put(DUPLICATE_TRANSACTION_COUNT, duplicateTransactionCount); From b4248a24dfa1fd74e022226ea7571a94b40633ef Mon Sep 17 00:00:00 2001 From: Dhaval Maniyar <100835289+logoutdhaval@users.noreply.github.com> Date: Mon, 27 Nov 2023 15:50:46 +0530 Subject: [PATCH 106/156] =?UTF-8?q?[PHEE-477]=20Create=20an=20orb=20that?= =?UTF-8?q?=20fetches=20the=20docker=20images=20and=20do=20helm=20upg?= =?UTF-8?q?=E2=80=A6=20(#130)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * PHEE-477 Create an orb that fetches the docker images and do helm upgrade * added github template --- .circleci/config.yml | 18 +++++++++++++++++- .github/pull_request_template.md | 22 ++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 .github/pull_request_template.md diff --git a/.circleci/config.yml b/.circleci/config.yml index ade07cb9..f0bdc9fd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,6 +60,13 @@ jobs: ./gradlew checkstyleMain ./gradlew bootJar docker build -t openmf/ph-ee-bulk-processor:latest . + if [ "$CIRCLE_BRANCH" != "master" ]; then + PR_NUMBER=$(basename $CIRCLE_PULL_REQUEST) + PR_TITLE=$(curl -sSL "https://api.github.com/repos/openmf/$CIRCLE_PR_REPONAME/pulls/$PR_NUMBER" | jq -r '.title') + JIRA_STORY=$(echo $PR_TITLE | cut -d "[" -f2 | cut -d "]" -f1 | tr '[A-Z]' '[a-z]') + if [ -z "$JIRA_STORY" ]; then echo "Invalid PR title" && exit 1; else echo "Ticket NO: $JIRA_STORY"; fi + docker image tag openmf/$CIRCLE_PR_REPONAME:latest openmf/$CIRCLE_PR_REPONAME:$JIRA_STORY + fi # Log in to DockerHub using environment variables - run: @@ -69,7 +76,16 @@ jobs: # Push the Docker image to DockerHub - run: name: Push Docker image to DockerHub - command: docker push openmf/ph-ee-bulk-processor:latest + command: | + if [ "$CIRCLE_BRANCH" = "master" ]; then + docker push openmf/ph-ee-bulk-processor:latest + fi + if [ "$CIRCLE_BRANCH" != "master" ]; then + PR_NUMBER=$(basename $CIRCLE_PULL_REQUEST) + PR_TITLE=$(curl -sSL "https://api.github.com/repos/openmf/$CIRCLE_PR_REPONAME/pulls/$PR_NUMBER" | jq -r '.title') + JIRA_STORY=$(echo $PR_TITLE | cut -d "[" -f2 | cut -d "]" -f1 | tr '[A-Z]' '[a-z]') + docker push openmf/$CIRCLE_PR_REPONAME:${JIRA_STORY} + fi workflows: version: 2 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..98a182a2 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,22 @@ +## Description + +* PR title should have jira ticket enclosed in `[]`.
+Format: ``` [jira_ticket] description```
+ex: [phee-123] PR title. +* Add a link to the Jira ticket. +* Describe the changes made and why they were made. + +## Checklist + +Please make sure these boxes are checked before submitting your pull request - thanks! +- [ ] Followed the PR title naming convention mentioned above. + +- [ ] Design related bullet points or design document link related to this PR added in the description above. + +- [ ] Updated corresponding Postman Collection or Api documentation for the changes in this PR. + +- [ ] Created/updated unit or integration tests for verifying the changes made. + +- [ ] Added required Swagger annotation and update API documentation with details of any API changes if applicable + +- [ ] Followed the naming conventions as given in https://docs.google.com/document/d/1Q4vaMSzrTxxh9TS0RILuNkSkYCxotuYk1Xe0CMIkkCU/edit?usp=sharing From 630ef6e297cd31227c50752a42bc1b81d841346f Mon Sep 17 00:00:00 2001 From: Dhaval Maniyar <100835289+logoutdhaval@users.noreply.github.com> Date: Mon, 18 Dec 2023 12:50:27 +0530 Subject: [PATCH 107/156] [PHEE-494] Add Minio to the ph ee engine helm chart (#132) * [PHEE-494] Add Minio to the ph ee engine helm chart * addressed review comments --- .../processor/bulk/camel/routes/BaseRouteBuilder.java | 2 +- .../processor/bulk/file/config/AwsStorageConfig.java | 8 +++++++- src/main/resources/application.yaml | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java index 9871445a..770f97dd 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BaseRouteBuilder.java @@ -29,7 +29,7 @@ public abstract class BaseRouteBuilder extends RouteBuilder { @Value("#{'${tenants}'.split(',')}") protected List tenants; - @Value("${cloud.aws.s3-base-url}") + @Value("${cloud.aws.s3BaseUrl}") protected String awsS3BaseUrl; public Logger logger = LoggerFactory.getLogger(this.getClass()); diff --git a/src/main/java/org/mifos/processor/bulk/file/config/AwsStorageConfig.java b/src/main/java/org/mifos/processor/bulk/file/config/AwsStorageConfig.java index 58f04c6a..25b1ebe2 100644 --- a/src/main/java/org/mifos/processor/bulk/file/config/AwsStorageConfig.java +++ b/src/main/java/org/mifos/processor/bulk/file/config/AwsStorageConfig.java @@ -3,6 +3,7 @@ import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import org.springframework.beans.factory.annotation.Value; @@ -21,12 +22,17 @@ public class AwsStorageConfig { @Value("${cloud.aws.region.static}") private String region; + @Value("${cloud.aws.s3BaseUrl}") + private String endpoint; @Bean @ConditionalOnProperty(value = "cloud.aws.enabled", havingValue = "true") public AmazonS3 s3Client() { AWSCredentials credentials = new BasicAWSCredentials(accessKey, accessSecret); - return AmazonS3ClientBuilder.standard().withCredentials(new AWSStaticCredentialsProvider(credentials)).withRegion(region).build(); + return AmazonS3ClientBuilder.standard().withCredentials(new AWSStaticCredentialsProvider(credentials)) + .withPathStyleAccessEnabled(true).withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, region)) + .build(); + } } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 879f43bf..cc748e19 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -49,7 +49,7 @@ channel: cloud: aws: enabled: true - s3-base-url: "https://paymenthub-ee.s3.ap-south-1.amazonaws.com" + s3BaseUrl: "https://s3.ap-south-1.amazonaws.com" credentials: access-key: ${AWS_ACCESS_KEY:access_key_from_aws} secret-key: ${AWS_SECRET_KEY:secret_key_from_aws} From ce14cca81aace4304c8228e0d4f7418f239f21f8 Mon Sep 17 00:00:00 2001 From: Dhaval Maniyar <100835289+logoutdhaval@users.noreply.github.com> Date: Tue, 26 Dec 2023 18:05:43 +0530 Subject: [PATCH 108/156] [PHEE-515] invalid bulk processor host name in service (#134) * [PHEE-515] invalid bulk processor host name in service * http to https --- src/main/resources/application.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index cc748e19..8a6982fe 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -152,7 +152,7 @@ identity_account_mapper: account_lookup_callback: /accountLookupCallback bulk_processor: - hostname : "http://ph-ee-connector-bulk:8080" + hostname : "https://ph-ee-connector-bulk:8443" csv: columnNames: "id,request_id,payment_mode,payer_identifier_type,payer_identifier,payee_identifier_type,payee_identifier,amount,currency,note,account_number,program_shortcode,cycle" From 4a9fd4cc24461e6e6510b15c4b03ddab68ba8ee6 Mon Sep 17 00:00:00 2001 From: Manoj <56669674+fynmanoj@users.noreply.github.com> Date: Wed, 27 Dec 2023 01:13:59 +0530 Subject: [PATCH 109/156] update-call-back-url --- src/main/resources/application.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 8a6982fe..7e62af70 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -193,7 +193,7 @@ payment-mode: endpoint: "bulk_connector_{MODE}-{dfspid}" debulkingDfspid: "lion" batch-authorization: - callback-url: "https://ph-ee-connector-bulk:80/authorization/callback" + callback-url: "https://ph-ee-connector-bulk:8443/authorization/callback" pubsub: room: From a60e3c4ba6f21859d7df9d0e61581e26454c5dbe Mon Sep 17 00:00:00 2001 From: Dhaval Maniyar <100835289+logoutdhaval@users.noreply.github.com> Date: Sun, 31 Dec 2023 15:17:02 +0530 Subject: [PATCH 110/156] [PHEE-522] Create Sync callback functionality in bulk (#136) * [PHEE-522] Create Sync callback functionality in bulk * callback port change * callback fix * callback issue fix --- .../api/definition/BatchTransactions.java | 4 +- .../BatchTransactionsController.java | 7 ++- .../bulk/camel/config/CamelProperties.java | 1 + .../camel/routes/BatchAggregateRoute.java | 7 ++- .../camel/routes/ProcessorStartRoute.java | 9 ++- .../bulk/camel/routes/SendCallbackRoute.java | 63 ++++++++++--------- .../processor/bulk/zeebe/ZeebeVariables.java | 2 +- .../bulk/zeebe/worker/AggregateWorker.java | 12 +++- .../bulk/zeebe/worker/SendCallbackWorker.java | 23 ++++--- 9 files changed, 81 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/api/definition/BatchTransactions.java b/src/main/java/org/mifos/processor/bulk/api/definition/BatchTransactions.java index 119ed31e..720bced0 100644 --- a/src/main/java/org/mifos/processor/bulk/api/definition/BatchTransactions.java +++ b/src/main/java/org/mifos/processor/bulk/api/definition/BatchTransactions.java @@ -1,5 +1,6 @@ package org.mifos.processor.bulk.api.definition; +import static org.mifos.processor.bulk.camel.config.CamelProperties.CALLBACK; import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_CLIENT_CORRELATION_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PROGRAM_ID; @@ -22,6 +23,7 @@ String batchTransactions(HttpServletRequest httpServletRequest, HttpServletRespo @RequestHeader(value = FILE_NAME, required = false) String fileName, @RequestHeader(value = PURPOSE) String purpose, @RequestHeader(value = HEADER_TYPE) String type, @RequestHeader(value = HEADER_PLATFORM_TENANT_ID) String tenant, @RequestHeader(value = HEADER_REGISTERING_INSTITUTE_ID, required = false) String registeringInstitutionId, - @RequestHeader(value = HEADER_PROGRAM_ID, required = false) String programId) throws IOException; + @RequestHeader(value = HEADER_PROGRAM_ID, required = false) String programId, + @RequestHeader(value = CALLBACK, required = false) String callbackUrl) throws IOException; } diff --git a/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java b/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java index 6a34612c..38129131 100644 --- a/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java +++ b/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java @@ -1,5 +1,6 @@ package org.mifos.processor.bulk.api.implementation; +import static org.mifos.processor.bulk.camel.config.CamelProperties.CALLBACK; import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PROGRAM_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; @@ -63,12 +64,14 @@ public class BatchTransactionsController implements BatchTransactions { @SneakyThrows @Override public String batchTransactions(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, String requestId, - String fileName, String purpose, String type, String tenant, String registeringInstitutionId, String programId) { + String fileName, String purpose, String type, String tenant, String registeringInstitutionId, String programId, + String callbackUrl) { log.info("Inside api logic"); Headers.HeaderBuilder headerBuilder = new Headers.HeaderBuilder().addHeader(HEADER_CLIENT_CORRELATION_ID, requestId) .addHeader(PURPOSE, purpose).addHeader(HEADER_TYPE, type).addHeader(HEADER_PLATFORM_TENANT_ID, tenant) - .addHeader(HEADER_REGISTERING_INSTITUTE_ID, registeringInstitutionId).addHeader(HEADER_PROGRAM_ID, programId); + .addHeader(HEADER_REGISTERING_INSTITUTE_ID, registeringInstitutionId).addHeader(HEADER_PROGRAM_ID, programId) + .addHeader(CALLBACK, callbackUrl); Optional validationResponse = isValidRequest(httpServletRequest, fileName, type); if (validationResponse.isPresent()) { diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index e4c71cd0..32ada255 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -83,4 +83,5 @@ private CamelProperties() {} public static final String EVENT_TYPE = "eventType"; public static final String DUPLICATE_TRANSACTION_LIST = "duplicateTransactionList"; public static final String ORIGINAL_TRANSACTION_LIST = "originalTransactionList"; + public static final String CALLBACK = "X-CallbackURL"; } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java index 5d597060..17b92c09 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java @@ -11,8 +11,10 @@ import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; +import org.apache.camel.ProducerTemplate; import org.apache.camel.model.dataformat.JsonLibrary; import org.mifos.processor.bulk.schema.BatchDTO; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -21,6 +23,8 @@ public class BatchAggregateRoute extends BaseRouteBuilder { @Value("${config.completion-threshold-check.completion-threshold}") private int completionThreshold; + @Autowired + private ProducerTemplate producerTemplate; @Override public void configure() throws Exception { @@ -52,7 +56,7 @@ public void configure() throws Exception { .log("Starting route direct:batch-aggregate-response-handler") // .setBody(exchange -> exchange.getIn().getBody(String.class)) .choice().when(header("CamelHttpResponseCode").isEqualTo("200")).log(LoggingLevel.INFO, "Batch summary request successful") - .unmarshal().json(JsonLibrary.Jackson, BatchDTO.class).process(exchange -> { + .log("Response body: ${body}").unmarshal().json(JsonLibrary.Jackson, BatchDTO.class).process(exchange -> { BatchDTO batchAggregateResponse = exchange.getIn().getBody(BatchDTO.class); int percentage = (int) (((double) batchAggregateResponse.getSuccessful() / batchAggregateResponse.getTotal()) * 100); @@ -62,6 +66,7 @@ public void configure() throws Exception { exchange.setProperty(COMPLETION_RATE, percentage); + producerTemplate.send(RouteId.SEND_CALLBACK.getValue(), exchange); }).otherwise().log(LoggingLevel.ERROR, "Batch aggregate request unsuccessful").process(exchange -> { exchange.setProperty(BATCH_STATUS_FAILED, true); exchange.setProperty(ERROR_DESCRIPTION, exchange.getIn().getBody(String.class)); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 374ddc67..64244f11 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -19,6 +19,7 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BULK_NOTIF_FAILURE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BULK_NOTIF_SUCCESS; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_URL; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CLIENT_CORRELATION_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_THRESHOLD; @@ -149,6 +150,8 @@ private void setup() { String requestId = exchange.getIn().getHeader("requestId", String.class); String purpose = exchange.getIn().getHeader("purpose", String.class); String batchId = UUID.randomUUID().toString(); + String callbackUrl = exchange.getIn().getHeader("X-CallbackURL", String.class); + exchange.setProperty(CALLBACK, callbackUrl); exchange.setProperty(BATCH_ID, batchId); exchange.setProperty(FILE_NAME, fileName); exchange.setProperty(REQUEST_ID, requestId); @@ -232,6 +235,7 @@ private void setup() { String requestId = exchange.getProperty(REQUEST_ID, String.class); String purpose = exchange.getProperty(PURPOSE, String.class); String batchId = exchange.getProperty(BATCH_ID, String.class); + String callbackUrl = exchange.getProperty(CALLBACK, String.class); String note = null; if (purpose == null || purpose.isEmpty()) { @@ -258,7 +262,6 @@ private void setup() { logger.debug("File uploaded {}", nm); // extracting and setting callback Url - String callbackUrl = exchange.getIn().getHeader("X-Callback-URL", String.class); exchange.setProperty(CALLBACK_URL, callbackUrl); List phases = phaseUtils.getValues(); @@ -269,7 +272,7 @@ private void setup() { variables.put(REQUEST_ID, requestId); variables.put(PURPOSE, purpose); variables.put(TENANT_ID, exchange.getProperty(TENANT_NAME)); - variables.put(CALLBACK_URL, callbackUrl); + variables.put(CALLBACK, callbackUrl); variables.put(PHASES, phases); variables.put(PHASE_COUNT, phases.size()); variables.put(NOTE, note); @@ -326,6 +329,7 @@ private void setup() { String clientCorrelationId = exchange.getIn().getHeader(HEADER_CLIENT_CORRELATION_ID, String.class); String registeringInstitutionId = exchange.getIn().getHeader(HEADER_REGISTERING_INSTITUTE_ID, String.class); String programId = exchange.getIn().getHeader(HEADER_PROGRAM_ID, String.class); + String callbackUrl = exchange.getIn().getHeader("X-CallbackURL", String.class); exchange.setProperty(FILE_NAME, filename); exchange.setProperty(REQUEST_ID, requestId); exchange.setProperty(PURPOSE, purpose); @@ -333,6 +337,7 @@ private void setup() { exchange.setProperty(CLIENT_CORRELATION_ID, clientCorrelationId); exchange.setProperty(REGISTERING_INSTITUTE_ID, registeringInstitutionId); exchange.setProperty(PROGRAM_ID, programId); + exchange.setProperty(CALLBACK, callbackUrl); }).choice().when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("raw")) .to("direct:start-batch-process-raw") .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("csv")) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java index 6810ca02..546a3cce 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java @@ -1,16 +1,16 @@ package org.mifos.processor.bulk.camel.routes; import static org.mifos.processor.bulk.camel.config.CamelProperties.CALLBACK_RESPONSE_CODE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_RETRY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_SUCCESS; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_URL; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_RATE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_CODE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_DESCRIPTION; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_CALLBACK_RETRY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASES; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASE_COUNT; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; @@ -34,55 +34,58 @@ public void configure() throws Exception { */ from(RouteId.SEND_CALLBACK.getValue()).id(RouteId.SEND_CALLBACK.getValue()).log("Starting route " + RouteId.SEND_CALLBACK.name()) - .log("Sending callback for Batch Processing").setBody(exchange -> { + .log("Sending callback for Batch Processing").setHeader(Exchange.HTTP_METHOD, constant("POST")).process(exchange -> { String body = String.format("The Batch Aggregation API was complete with : %s", exchange.getProperty(COMPLETION_RATE).toString()); - callbackUrl = exchange.getProperty(CALLBACK_URL).toString(); - return body; - }).to("direct:set-jws-signature").toD("${header.callbackUrl}?bridgeEndpoint=true&throwExceptionOnFailure=false").choice() - .when(header("CamelHttpResponseCode").startsWith("2")).log(LoggingLevel.INFO, "Callback sending was successful") - .process(exchange -> { - List phases = (List) exchange.getProperty(PHASES); + callbackUrl = exchange.getProperty(CALLBACK, String.class); + logger.info("Callback URL: {}", callbackUrl); + logger.info("Callback Body: {}", body); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonString = objectMapper.writeValueAsString(body); + exchange.getIn().setBody(jsonString); + }).setHeader(Exchange.HTTP_METHOD, constant("POST")) + .toD("${exchangeProperty.X-CallbackURL}?bridgeEndpoint=true&throwExceptionOnFailure=false") + .log(LoggingLevel.INFO, "Callback Response body: ${body}").choice() + .when(header(Exchange.HTTP_RESPONSE_CODE).regex("^2\\d{2}$")).log(LoggingLevel.INFO, "Callback sending was successful") + .process(exchange -> { + List phases = exchange.getProperty(PHASES, List.class); exchange.setProperty(CALLBACK_RESPONSE_CODE, exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE)); exchange.setProperty(CALLBACK_RETRY, 1); exchange.setProperty(CALLBACK_SUCCESS, true); - eliminatePhases(exchange, phases, (Integer) exchange.getProperty(PHASE_COUNT), - (Integer) exchange.getProperty(COMPLETION_RATE)); - + eliminatePhases(exchange); }).otherwise().log(LoggingLevel.ERROR, "Callback request was unsuccessful").process(exchange -> { - if (exchange.getProperty(CALLBACK_RETRY).equals(exchange.getProperty(MAX_CALLBACK_RETRY))) { - List phases = (List) exchange.getProperty(PHASES); + int retry = exchange.getProperty(CALLBACK_RETRY, Integer.class); + int maxRetry = exchange.getProperty(MAX_CALLBACK_RETRY, Integer.class); + if (retry >= maxRetry) { + List phases = exchange.getProperty(PHASES, List.class); logger.info("Retry Exhausted, setting Callback as Failed"); - eliminatePhases(exchange, phases, (Integer) exchange.getProperty(PHASE_COUNT), - (Integer) exchange.getProperty(COMPLETION_RATE)); + eliminatePhases(exchange); exchange.setProperty(CALLBACK_RESPONSE_CODE, exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE)); + exchange.setProperty(CALLBACK_SUCCESS, false); + exchange.setProperty(ERROR_DESCRIPTION, exchange.getIn().getBody(String.class)); + exchange.setProperty(ERROR_CODE, exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE)); } else { - int retry = (int) exchange.getProperty(CALLBACK_RETRY); retry++; - logger.info("Retry Left {}, Setting Callback as Failed and Retrying...", - ((int) exchange.getProperty(MAX_CALLBACK_RETRY) - retry)); + logger.info("Retry Left {}, Setting Callback as Failed and Retrying...", (maxRetry - retry)); exchange.setProperty(CALLBACK_RETRY, retry); - exchange.setProperty(CALLBACK_RESPONSE_CODE, exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE)); } + exchange.setProperty(CALLBACK_RESPONSE_CODE, exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE)); exchange.setProperty(CALLBACK_SUCCESS, false); exchange.setProperty(ERROR_DESCRIPTION, exchange.getIn().getBody(String.class)); exchange.setProperty(ERROR_CODE, exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE)); - }); + }); } - public void eliminatePhases(Exchange exchange, List phases, int phaseCount, int completionRate) { - int i = 0; - while (phases.size() > 0 && phases.size() > i) { - if (phases.get(i) <= completionRate) { - phases.remove(i); - } - i++; - } + public void eliminatePhases(Exchange exchange) { + List phases = exchange.getProperty(PHASES, List.class); + int completionRate = exchange.getProperty(COMPLETION_RATE, Integer.class); + + phases.removeIf(phase -> phase <= completionRate); + exchange.setProperty(PHASES, phases); - exchange.setProperty(PHASE_COUNT, phaseCount); } } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index b08d94ee..a3c8168d 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -107,7 +107,7 @@ private ZeebeVariables() {} public static final String CALLBACK_SUCCESS = "callbackSuccessful"; - public static final String CALLBACK_URL = "callbackUrl"; + public static final String CALLBACK_URL = "X-CallbackURL"; public static final String MAX_CALLBACK_RETRY = "maxCallbackRetry"; diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java index c281de23..3c30f9fc 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java @@ -2,9 +2,14 @@ import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_STATUS_FAILED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_RETRY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_RATE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_CODE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_DESCRIPTION; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_CALLBACK_RETRY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASES; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASE_COUNT; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.RETRY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; @@ -31,7 +36,12 @@ public void setup() { Exchange exchange = new DefaultExchange(camelContext); exchange.setProperty(BATCH_ID, variables.get(BATCH_ID)); exchange.setProperty(TENANT_ID, variables.get(TENANT_ID)); - + exchange.setProperty(MAX_CALLBACK_RETRY, variables.get(MAX_CALLBACK_RETRY)); + exchange.setProperty(CALLBACK_RETRY, variables.getOrDefault(CALLBACK_RETRY, 0)); + exchange.setProperty(CALLBACK, variables.get(CALLBACK)); + // exchange.setProperty(COMPLETION_RATE, variables.get(COMPLETION_RATE)); + exchange.setProperty(PHASES, variables.get(PHASES)); + exchange.setProperty(PHASE_COUNT, variables.get(PHASE_COUNT)); sendToCamelRoute(RouteId.BATCH_AGGREGATE, exchange); Boolean batchStatusFailed = exchange.getProperty(BATCH_STATUS_FAILED, Boolean.class); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java index cb362b06..a34df04b 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java @@ -1,9 +1,9 @@ package org.mifos.processor.bulk.zeebe.worker; import static org.mifos.processor.bulk.camel.config.CamelProperties.CALLBACK_RESPONSE_CODE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_RETRY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_SUCCESS; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_URL; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_RATE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_CODE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_DESCRIPTION; @@ -30,14 +30,19 @@ public void setup() { int retry = variables.getOrDefault(CALLBACK_RETRY, 0).equals(variables.get(MAX_STATUS_RETRY)) ? 0 : (int) variables.getOrDefault(CALLBACK_RETRY, 0); Exchange exchange = new DefaultExchange(camelContext); - exchange.setProperty(MAX_CALLBACK_RETRY, variables.get(MAX_CALLBACK_RETRY)); - exchange.setProperty(CALLBACK_RETRY, retry); - exchange.setProperty(CALLBACK_URL, variables.get(CALLBACK_URL)); - exchange.setProperty(COMPLETION_RATE, variables.get(COMPLETION_RATE)); - exchange.setProperty(PHASES, variables.get(PHASES)); - exchange.setProperty(PHASE_COUNT, variables.get(PHASE_COUNT)); - sendToCamelRoute(RouteId.SEND_CALLBACK, exchange); - + if (variables.get(CALLBACK_RETRY) != null && variables.get(CALLBACK_RETRY).equals(variables.get(MAX_CALLBACK_RETRY))) { + exchange.setProperty(CALLBACK_SUCCESS, false); + exchange.setProperty(CALLBACK_RESPONSE_CODE, variables.get(CALLBACK_RESPONSE_CODE)); + } else { + exchange = new DefaultExchange(camelContext); + exchange.setProperty(MAX_CALLBACK_RETRY, variables.get(MAX_CALLBACK_RETRY)); + exchange.setProperty(CALLBACK_RETRY, variables.getOrDefault(CALLBACK_RETRY, 0)); + exchange.setProperty(CALLBACK, variables.get(CALLBACK)); + exchange.setProperty(COMPLETION_RATE, variables.get(COMPLETION_RATE)); + exchange.setProperty(PHASES, variables.get(PHASES)); + exchange.setProperty(PHASE_COUNT, variables.get(PHASE_COUNT)); + sendToCamelRoute(RouteId.SEND_CALLBACK, exchange); + } Boolean callbackSuccess = exchange.getProperty(CALLBACK_SUCCESS, Boolean.class); if (callbackSuccess == null || !callbackSuccess) { variables.put(ERROR_CODE, exchange.getProperty(ERROR_CODE)); From 74ddf3aac9deae38c0f65d4b4dd68f85bb72df86 Mon Sep 17 00:00:00 2001 From: Apurb Rajdhan <103931815+apurbraj@users.noreply.github.com> Date: Tue, 2 Jan 2024 13:01:55 +0530 Subject: [PATCH 111/156] [GOV-324] Batch Account Lookup (#115) * GOV-324 * adding payee dfsp id * Batch splitting based on payee dfsp id * Adding logs * changing tenant * Modify if condition * Adding logs --- build.gradle | 1 + .../routes/AccountLookupCallbackRoute.java | 111 ++++++++++++++++++ .../bulk/camel/routes/AccountLookupRoute.java | 46 ++++++++ .../bulk/camel/routes/InitSubBatchRoute.java | 5 +- .../camel/routes/ProcessorStartRoute.java | 18 ++- .../processor/bulk/camel/routes/RouteId.java | 5 +- .../bulk/camel/routes/SplittingRoute.java | 92 +++++++++++---- .../bulk/properties/TenantImplementation.java | 26 ++++ .../TenantImplementationProperties.java | 21 ++++ .../schema/BatchAccountLookupResponseDTO.java | 18 +++ .../processor/bulk/schema/BeneficiaryDTO.java | 19 +++ .../processor/bulk/schema/Transaction.java | 5 +- .../processor/bulk/zeebe/ZeebeVariables.java | 8 +- .../zeebe/worker/AccountLookupWorker.java | 2 - .../BatchAccountLookupCallbackWorker.java | 44 +++++++ .../worker/BatchAccountLookupWorker.java | 68 +++++++++++ .../bulk/zeebe/worker/SplittingWorker.java | 3 + .../processor/bulk/zeebe/worker/Worker.java | 5 +- src/main/resources/application.yaml | 15 +++ 19 files changed, 475 insertions(+), 37 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupCallbackRoute.java create mode 100644 src/main/java/org/mifos/processor/bulk/properties/TenantImplementation.java create mode 100644 src/main/java/org/mifos/processor/bulk/properties/TenantImplementationProperties.java create mode 100644 src/main/java/org/mifos/processor/bulk/schema/BatchAccountLookupResponseDTO.java create mode 100644 src/main/java/org/mifos/processor/bulk/schema/BeneficiaryDTO.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupCallbackWorker.java create mode 100644 src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupWorker.java diff --git a/build.gradle b/build.gradle index 23ed3b42..a11df0f2 100644 --- a/build.gradle +++ b/build.gradle @@ -108,6 +108,7 @@ dependencies { // miscellaneous test dependency testImplementation "com.google.truth:truth:1.1.3" testImplementation 'com.google.code.gson:gson:2.9.0' + implementation 'io.rest-assured:rest-assured:4.4.0' } configure(this) { diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupCallbackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupCallbackRoute.java new file mode 100644 index 00000000..216d051c --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupCallbackRoute.java @@ -0,0 +1,111 @@ +package org.mifos.processor.bulk.camel.routes; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; +import static org.mifos.processor.bulk.camel.config.CamelProperties.OVERRIDE_HEADER; +import static org.mifos.processor.bulk.camel.config.CamelProperties.RESULT_TRANSACTION_LIST; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_SUCCESSFUL_TRANSACTION_AMOUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_SUCCESSFUL_TRANSACTION_COUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.RESULT_FILE; + +import io.camunda.zeebe.client.ZeebeClient; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import org.mifos.processor.bulk.schema.BatchAccountLookupResponseDTO; +import org.mifos.processor.bulk.schema.BeneficiaryDTO; +import org.mifos.processor.bulk.schema.Transaction; +import org.mifos.processor.bulk.schema.TransactionResult; +import org.mifos.processor.bulk.utility.Utils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class AccountLookupCallbackRoute extends BaseRouteBuilder { + + @Autowired + private ZeebeClient zeebeClient; + private Integer totalApprovedAmount; + private Integer totalApprovedCount; + + @Override + public void configure() throws Exception { + from("direct:accountLookupCallback").id("direct:accountLookupCallback") + .log("Starting route " + RouteId.ACCOUNT_LOOKUP_CALLBACK.name()).to("direct:download-file") + .to("direct:get-transaction-array").to("direct:batch-account-lookup-callback") + .process(exchange -> exchange.setProperty(OVERRIDE_HEADER, true)); + from("direct:batch-account-lookup-callback").id("direct:batch-account-lookup-callback").process(exchange -> { + String serverFileName = exchange.getProperty(SERVER_FILE_NAME, String.class); + String resultFile = String.format("Result_%s", serverFileName); + BatchAccountLookupResponseDTO batchAccountLookupCallback = objectMapper + .readValue(exchange.getProperty("batchAccountLookupCallback", String.class), BatchAccountLookupResponseDTO.class); + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + List transactionResultList = new ArrayList<>(); + List updatedTransactionList = new ArrayList<>(); + Map variables = new HashMap<>(); + + updateTransactionStatus(transactionList, batchAccountLookupCallback.getBeneficiaryDTOList(), transactionResultList, + updatedTransactionList); + exchange.setProperty(PARTY_LOOKUP_SUCCESSFUL_TRANSACTION_AMOUNT, totalApprovedAmount); + exchange.setProperty(PARTY_LOOKUP_SUCCESSFUL_TRANSACTION_COUNT, totalApprovedCount); + exchange.setProperty(RESULT_TRANSACTION_LIST, transactionResultList); + exchange.setProperty(RESULT_FILE, resultFile); + exchange.setProperty(TRANSACTION_LIST, updatedTransactionList); + Long workflowInstanceKey = Long.valueOf(exchange.getProperty("workflowInstanceKey").toString()); + variables.put(PARTY_LOOKUP_SUCCESSFUL_TRANSACTION_AMOUNT, totalApprovedAmount); + variables.put(PARTY_LOOKUP_SUCCESSFUL_TRANSACTION_COUNT, totalApprovedCount); + if (zeebeClient != null) { + + zeebeClient.newSetVariablesCommand(workflowInstanceKey).variables(variables).send().join(); + } + }) + // setting localfilepath as result file to make sure result file is uploaded + .log("updating orignal").setProperty(LOCAL_FILE_PATH, exchangeProperty(SERVER_FILE_NAME)) + .setProperty(OVERRIDE_HEADER, constant(true)).to("direct:update-file").to("direct:upload-file") + .log("updating failed transaction").setProperty(TRANSACTION_LIST, exchangeProperty(RESULT_TRANSACTION_LIST)) + .setProperty(LOCAL_FILE_PATH, exchangeProperty(RESULT_FILE)).setProperty(OVERRIDE_HEADER, constant(true)) + .to("direct:update-result-file").to("direct:upload-file"); + } + + public List updateTransactionStatus(List transactionList, + List batchAccountLookupResponseDTO, List transactionResultList, + List updatedTransactionList) { + totalApprovedCount = 0; + totalApprovedAmount = 0; + AtomicInteger count = new AtomicInteger(totalApprovedCount); + AtomicInteger amount = new AtomicInteger(totalApprovedAmount); + + transactionList.forEach(transaction -> { + Optional matchingBeneficiary = batchAccountLookupResponseDTO.stream() + .filter(beneficiary -> transaction.getPayeeIdentifier().equals(beneficiary.getPayeeIdentity())).findFirst(); + + if (matchingBeneficiary.isPresent()) { + count.incrementAndGet(); // Increment the count atomically + try { + amount.addAndGet(Integer.parseInt(transaction.getAmount())); + } catch (NumberFormatException e) { + logger.error(e.getMessage()); + } + String identifier = matchingBeneficiary.get().getFinancialAddress(); + transaction.setPayeeIdentifier(identifier); + transaction.setPayeeDfspId(matchingBeneficiary.get().getBankingInstitutionCode()); + updatedTransactionList.add(transaction); + } else { + TransactionResult transactionResult = Utils.mapToResultDTO(transaction); + transactionResult.setErrorCode("404"); + transactionResult.setErrorDescription("Payee Identifier not found"); + transactionResult.setStatus("Failed"); + transactionResultList.add(transactionResult); + } + }); + totalApprovedCount = count.get(); + totalApprovedAmount = amount.get(); + + return transactionResultList; + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java index 15836143..2b5c55c3 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java @@ -1,13 +1,24 @@ package org.mifos.processor.bulk.camel.routes; import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.OVERRIDE_HEADER; import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_IDENTITY; import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYMENT_MODALITY; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REGISTERING_INSTITUTION_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import javax.net.ssl.HttpsURLConnection; import org.apache.camel.Exchange; import org.apache.camel.Processor; +import org.mifos.connector.common.identityaccountmapper.dto.AccountMapperRequestDTO; +import org.mifos.connector.common.identityaccountmapper.dto.BeneficiaryDTO; +import org.mifos.processor.bulk.schema.Transaction; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -18,6 +29,12 @@ public class AccountLookupRoute extends BaseRouteBuilder { private String accountLookupEndpoint; @Value("${identity_account_mapper.hostname}") private String identityURL; + @Value("${identity_account_mapper.hostname}") + private String identityMapperURL; + @Value("${identity_account_mapper.batch_account_lookup_callback}") + private String batchAccountLookupCallback; + @Value("${identity_account_mapper.batch_account_lookup}") + private String batchAccountLookup; @Override public void configure() throws Exception { @@ -38,5 +55,34 @@ public void process(Exchange exchange) throws Exception { .toD(identityURL + accountLookupEndpoint + "?" + PAYEE_IDENTITY + "=${exchangeProperty.payeeIdentity}&" + PAYMENT_MODALITY + "=${exchangeProperty.paymentModality}&" + "requestId=${exchangeProperty.requestId}") .log("API Response: ${body}").process(disableSslProcessor); + + from(RouteId.ACCOUNT_LOOKUP.getValue()).id(RouteId.ACCOUNT_LOOKUP.getValue()).log("Starting route " + RouteId.ACCOUNT_LOOKUP.name()) + .process(exchange -> exchange.setProperty(OVERRIDE_HEADER, true)).to("direct:download-file") + .to("direct:get-transaction-array").to("direct:batch-account-lookup").to("direct:update-file").to("direct:upload-file") + .process(exchange -> { + exchange.setProperty(PARTY_LOOKUP_FAILED, false); + }); + + from("direct:batch-account-lookup").id("direct:batch-account-lookup").process(exchange -> { + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + HashMap> stringListHashMap = new HashMap<>(); + List beneficiaryDTOList = new ArrayList<>(); + transactionList.forEach(transaction -> { + beneficiaryDTOList.add(new BeneficiaryDTO(transaction.getPayeeIdentifier(), "", "", "")); + }); + String requestId = exchange.getProperty(REQUEST_ID, String.class); + String callbackUrl = exchange.getProperty(CALLBACK, String.class); + String registeringInstitutionId = exchange.getProperty(HEADER_REGISTERING_INSTITUTE_ID, String.class); + AccountMapperRequestDTO accountMapperRequestDTO = new AccountMapperRequestDTO(requestId, registeringInstitutionId, + beneficiaryDTOList); + String requestBody = objectMapper.writeValueAsString(accountMapperRequestDTO); + + exchange.getIn().setHeader(CALLBACK, callbackUrl); + exchange.getIn().setHeader(REGISTERING_INSTITUTION_ID, registeringInstitutionId); + exchange.getIn().setHeader("Content-type", "application/json"); + exchange.getIn().setBody(requestBody); + }).setHeader(Exchange.HTTP_METHOD, constant("POST")).toD(identityURL + batchAccountLookup).log("API Response: ${body}") + .process(disableSslProcessor); + } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index 96512210..8983c5bf 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -128,10 +128,13 @@ public void configure() throws Exception { int index = exchange.getProperty(Exchange.LOOP_INDEX, Integer.class); List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); Transaction transaction = transactionList.get(index); + exchange.setProperty(REQUEST_ID, transaction.getRequestId()); + exchange.setProperty("payeeDFSPId", transaction.getPayeeDfspId()); logger.info("REQUEST_ID: {}", transaction.getRequestId()); exchange.setProperty(TRANSACTION_LIST_ELEMENT, transaction); - }).setHeader("Platform-TenantId", exchangeProperty(TENANT_NAME)).to("direct:dynamic-payload-setter") + }).setHeader("Platform-TenantId", exchangeProperty(TENANT_NAME)) + .setHeader("X-PayeeDFSP-ID", exchangeProperty("payeeDFSPId")).to("direct:dynamic-payload-setter") .to("direct:external-api-call").to("direct:external-api-response-handler").end() // end loop block .endChoice(); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 64244f11..0a508899 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -67,6 +67,8 @@ import org.mifos.processor.bulk.config.Program; import org.mifos.processor.bulk.config.RegisteringInstitutionConfig; import org.mifos.processor.bulk.file.FileTransferService; +import org.mifos.processor.bulk.properties.TenantImplementation; +import org.mifos.processor.bulk.properties.TenantImplementationProperties; import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.utility.PhaseUtils; import org.mifos.processor.bulk.utility.Utils; @@ -85,6 +87,8 @@ public class ProcessorStartRoute extends BaseRouteBuilder { @Autowired private ZeebeProcessStarter zeebeProcessStarter; + @Autowired + TenantImplementationProperties tenantImplementationProperties; @Autowired @Qualifier("awsStorage") @@ -288,9 +292,10 @@ private void setup() { log.debug("Variables published to zeebe: {}", variables); JSONObject response = new JSONObject(); + String bpmn = getWorkflowForTenant(exchange.getProperty(TENANT_NAME).toString(), "batch-transactions"); try { - String tenantSpecificWorkflowId = workflowId.replace("{dfspid}", exchange.getProperty(TENANT_NAME).toString()); + String tenantSpecificWorkflowId = bpmn.replace("{dfspid}", exchange.getProperty(TENANT_NAME).toString()); String txnId = zeebeProcessStarter.startZeebeWorkflow(tenantSpecificWorkflowId, "", variables); if (txnId == null || txnId.isEmpty()) { response.put("errorCode", 500); @@ -328,6 +333,7 @@ private void setup() { String type = exchange.getIn().getHeader("Type", String.class); String clientCorrelationId = exchange.getIn().getHeader(HEADER_CLIENT_CORRELATION_ID, String.class); String registeringInstitutionId = exchange.getIn().getHeader(HEADER_REGISTERING_INSTITUTE_ID, String.class); + logger.info("registeringInstitutionId {}", registeringInstitutionId); String programId = exchange.getIn().getHeader(HEADER_PROGRAM_ID, String.class); String callbackUrl = exchange.getIn().getHeader("X-CallbackURL", String.class); exchange.setProperty(FILE_NAME, filename); @@ -413,6 +419,16 @@ private boolean verifyData(File file) throws IOException { return true; } + public String getWorkflowForTenant(String tenantId, String useCase) { + + for (TenantImplementation tenant : tenantImplementationProperties.getTenants()) { + if (tenant.getId().equals(tenantId)) { + return tenant.getFlows().getOrDefault(useCase, "default"); + } + } + return "default"; + } + private boolean verifyRow(String[] row) { for (int i = 1; i < row.length; i++) { row[i] = row[i].trim(); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java b/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java index 1fae2316..3a4879b3 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/RouteId.java @@ -4,8 +4,9 @@ public enum RouteId { PARTY_LOOKUP("direct:partyLookup"), APPROVAL("direct:approval"), ORDERING("direct:ordering"), SPLITTING("direct:splitting"), FORMATTING( "direct:formatting"), BATCH_STATUS("direct:batchStatus"), SEND_CALLBACK("direct:sendCallback"), MERGE_BACK( - "direct:mergeSubBatch"), INIT_SUB_BATCH( - "direct:init-sub-batches"), BATCH_AGGREGATE("direct:batch-aggregate"), DE_DUPLICATION("direct:deDuplication"); + "direct:mergeSubBatch"), INIT_SUB_BATCH("direct:init-sub-batches"), BATCH_AGGREGATE( + "direct:batch-aggregate"), DE_DUPLICATION("direct:deDuplication"), ACCOUNT_LOOKUP( + "direct:accountLookup"), ACCOUNT_LOOKUP_CALLBACK("direct:accountLookupCallback"); private final String value; diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java index 3dce4aec..8edfd624 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java @@ -16,17 +16,25 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_FAILED; +import com.fasterxml.jackson.databind.SequenceWriter; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; import java.io.BufferedReader; +import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; import org.apache.camel.LoggingLevel; import org.mifos.processor.bulk.schema.SubBatchEntity; import org.mifos.processor.bulk.schema.Transaction; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -35,6 +43,10 @@ public class SplittingRoute extends BaseRouteBuilder { @Value("${config.splitting.sub-batch-size}") private int subBatchSize; + @Autowired + private CsvMapper csvMapper; + @Value("${config.partylookup.enable}") + private boolean partyLookupEnabled; @Override public void configure() throws Exception { @@ -44,7 +56,7 @@ public void configure() throws Exception { * direct:create-sub-batch-file 2. direct:upload-sub-batch-file */ from(RouteId.SPLITTING.getValue()).id(RouteId.SPLITTING.getValue()).log("Starting route " + RouteId.SPLITTING.name()) - .to("direct:download-file").to("direct:create-sub-batch-file").choice() + .to("direct:download-file").to("direct:get-transaction-array").to("direct:create-sub-batch-file").choice() .when(exchange -> exchange.getProperty(SUB_BATCH_CREATED, Boolean.class)).to("direct:upload-sub-batch-file").otherwise() .log("No sub batch created, so skipping upload").end().process(exchange -> exchange.setProperty(SPLITTING_FAILED, false)); @@ -53,33 +65,63 @@ public void configure() throws Exception { String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); BufferedReader reader = new BufferedReader(new FileReader(filepath)); String header = reader.readLine() + System.lineSeparator(); - List lines = new ArrayList<>(); - String line = null; - while ((line = reader.readLine()) != null) { - lines.add(line); - } - reader.close(); + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + List subBatchFile = new ArrayList<>(); + Set distinctPayeeIds = transactionList.stream().map(Transaction::getPayeeDfspId).collect(Collectors.toSet()); + logger.info("Payee id {}", distinctPayeeIds); + if (partyLookupEnabled && !distinctPayeeIds.isEmpty()) { + // Create a map to store transactions for each payeeid + Map> transactionsByPayeeId = new HashMap<>(); + + // Split the list based on distinct payeeids + for (String payeeId : distinctPayeeIds) { + List transactionsForPayee = transactionList.stream() + .filter(transaction -> payeeId.equals(transaction.getPayeeDfspId())).collect(Collectors.toList()); + + transactionsByPayeeId.put(payeeId, transactionsForPayee); + } - if (lines.size() <= subBatchSize) { - exchange.setProperty(SUB_BATCH_CREATED, false); - exchange.setProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, new ArrayList()); - logger.info("Skipping creating sub batch, as batch size is less than configured sub-batch size"); - return; - } + for (String payeeId : distinctPayeeIds) { + List transactionsForSpecificPayee = transactionsByPayeeId.get(payeeId); + String filename = UUID.randomUUID() + "_" + "sub-batch-" + payeeId + ".csv"; + logger.info("Created sub-batch with file name {}", filename); + CsvSchema csvSchema = csvMapper.schemaFor(Transaction.class); + csvSchema = csvSchema.withHeader(); + File file = new File(filename); + SequenceWriter writer = csvMapper.writerWithSchemaFor(Transaction.class).with(csvSchema).writeValues(file); + for (Transaction transaction : transactionsForSpecificPayee) { + writer.write(transaction); + } + subBatchFile.add(filename); + } + } else { + List lines = new ArrayList<>(); + String line = null; + while ((line = reader.readLine()) != null) { + lines.add(line); + } + reader.close(); - List subBatchFile = new ArrayList<>(); - int subBatchCount = 1; - for (int i = 0; i < lines.size(); i += subBatchSize) { - String filename = UUID.randomUUID() + "_" + "sub-batch-" + subBatchCount + ".csv"; - FileWriter writer = new FileWriter(filename); - writer.write(header); - for (int j = i; j < Math.min(i + subBatchSize, lines.size()); j++) { - writer.write(lines.get(j) + System.lineSeparator()); + if (lines.size() <= subBatchSize) { + exchange.setProperty(SUB_BATCH_CREATED, false); + exchange.setProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, new ArrayList()); + logger.info("Skipping creating sub batch, as batch size is less than configured sub-batch size"); + return; + } + + int subBatchCount = 1; + for (int i = 0; i < lines.size(); i += subBatchSize) { + String filename = UUID.randomUUID() + "_" + "sub-batch-" + subBatchCount + ".csv"; + FileWriter writer = new FileWriter(filename); + writer.write(header); + for (int j = i; j < Math.min(i + subBatchSize, lines.size()); j++) { + writer.write(lines.get(j) + System.lineSeparator()); + } + writer.close(); + logger.info("Created sub-batch with file name {}", filename); + subBatchFile.add(filename); + subBatchCount++; } - writer.close(); - logger.info("Created sub-batch with file name {}", filename); - subBatchFile.add(filename); - subBatchCount++; } exchange.setProperty(SUB_BATCH_FILE_ARRAY, subBatchFile); exchange.setProperty(SUB_BATCH_COUNT, subBatchFile.size()); diff --git a/src/main/java/org/mifos/processor/bulk/properties/TenantImplementation.java b/src/main/java/org/mifos/processor/bulk/properties/TenantImplementation.java new file mode 100644 index 00000000..4d808354 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/properties/TenantImplementation.java @@ -0,0 +1,26 @@ +package org.mifos.processor.bulk.properties; + +import java.util.HashMap; + +public class TenantImplementation { + + String id; + HashMap flows; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public HashMap getFlows() { + return flows; + } + + public void setFlows(HashMap flows) { + this.flows = flows; + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/properties/TenantImplementationProperties.java b/src/main/java/org/mifos/processor/bulk/properties/TenantImplementationProperties.java new file mode 100644 index 00000000..4c1a2fd3 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/properties/TenantImplementationProperties.java @@ -0,0 +1,21 @@ +package org.mifos.processor.bulk.properties; + +import java.util.List; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@ConfigurationProperties(prefix = "bpmns") +public class TenantImplementationProperties { + + List tenants; + + public List getTenants() { + return tenants; + } + + public void setTenants(List tenants) { + this.tenants = tenants; + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/schema/BatchAccountLookupResponseDTO.java b/src/main/java/org/mifos/processor/bulk/schema/BatchAccountLookupResponseDTO.java new file mode 100644 index 00000000..939fda60 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/BatchAccountLookupResponseDTO.java @@ -0,0 +1,18 @@ +package org.mifos.processor.bulk.schema; + +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class BatchAccountLookupResponseDTO { + + private String requestId; + private String registeringInstitutionId; + private List beneficiaryDTOList; +} diff --git a/src/main/java/org/mifos/processor/bulk/schema/BeneficiaryDTO.java b/src/main/java/org/mifos/processor/bulk/schema/BeneficiaryDTO.java new file mode 100644 index 00000000..b82db523 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/BeneficiaryDTO.java @@ -0,0 +1,19 @@ +package org.mifos.processor.bulk.schema; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class BeneficiaryDTO { + + private String payeeIdentity; + private String paymentModality; + private String financialAddress; + private String bankingInstitutionCode; + +} diff --git a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java index e49eb3e5..7c8f9514 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java +++ b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java @@ -11,7 +11,7 @@ @Setter @JsonIgnoreProperties(ignoreUnknown = true) @JsonPropertyOrder({ "id", "request_id", "payment_mode", "account_number", "payer_identifier_type", "payer_identifier", - "payee_identifier_type", "payee_identifier", "amount", "currency", "note", "program_shortcode", "cycle" }) + "payee_identifier_type", "payee_identifier", "amount", "currency", "note", "program_shortcode", "cycle", "payee_dfsp_id" }) public class Transaction implements CsvSchema { @JsonProperty("id") @@ -53,6 +53,9 @@ public class Transaction implements CsvSchema { @JsonProperty("cycle") private String cycle; + @JsonProperty("payee_dfsp_id") + private String payeeDfspId; + @JsonIgnore private String batchId; diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index a3c8168d..83fdf99e 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -153,9 +153,6 @@ private ZeebeVariables() {} public static final String AUTHORIZATION_FAIL_REASON = "authorizationFailReason"; public static final String PAYER_IDENTIFIER = "payerIdentifier"; - - public static final String PARTY_LOOKUP_SUCCESSFUL_TRANSACTION_COUNT = "partyLookupSuccessfulTransactionAmount"; - public static final String CURRENCY = "currency"; public static final String AUTHORIZATION_RESPONSE = "authorizationResponse"; @@ -164,4 +161,9 @@ private ZeebeVariables() {} public static final String DUPLICATE_TRANSACTION_COUNT = "duplicateTransactionCount"; + public static final String BATCH_ACCOUNT_LOOKUP_RESPONSE = "batchAccountLookupResponse"; + public static final String PARTY_LOOKUP_SUCCESSFUL_TRANSACTION_AMOUNT = "totalApprovedAmount"; + public static final String PARTY_LOOKUP_SUCCESSFUL_TRANSACTION_COUNT = "totalApprovedCount"; + public static final String REGISTERING_INSTITUTION_ID = "X-Registering-Institution-ID"; + } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java index d682dabb..6107c604 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AccountLookupWorker.java @@ -44,8 +44,6 @@ public class AccountLookupWorker extends BaseWorker { private String identityMapperURL; @Value("${identity_account_mapper.account_lookup_callback}") private String accountLookupCallback; - @Value("${bulk_processor.hostname}") - private String bulkURL; @Value("${identity_account_mapper.account_lookup}") private String accountLookupEndpoint; diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupCallbackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupCallbackWorker.java new file mode 100644 index 00000000..4a94084e --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupCallbackWorker.java @@ -0,0 +1,44 @@ +package org.mifos.processor.bulk.zeebe.worker; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_FAILED; +import static org.mifos.processor.bulk.zeebe.worker.Worker.BATCH_ACCOUNT_LOOKUP_CALLBACK; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.zeebe.client.ZeebeClient; +import java.util.Map; +import org.apache.camel.CamelContext; +import org.apache.camel.Exchange; +import org.apache.camel.support.DefaultExchange; +import org.mifos.processor.bulk.camel.routes.RouteId; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class BatchAccountLookupCallbackWorker extends BaseWorker { + + @Autowired + private ZeebeClient zeebeClient; + @Autowired + private CamelContext camelContext; + @Autowired + private ObjectMapper objectMapper; + + @Override + public void setup() { + logger.info("## generating " + BATCH_ACCOUNT_LOOKUP_CALLBACK + "zeebe worker"); + newWorker(BATCH_ACCOUNT_LOOKUP_CALLBACK, ((client, job) -> { + Map variables = job.getVariablesAsMap(); + Exchange exchange = new DefaultExchange(camelContext); + String filename = (String) variables.get(FILE_NAME); + String batchAccountLookupCallback = (String) variables.get("batchAccountLookupCallback"); + variables.put(PARTY_LOOKUP_FAILED, false); + exchange.setProperty(SERVER_FILE_NAME, filename); + exchange.setProperty("batchAccountLookupCallback", batchAccountLookupCallback); + exchange.setProperty("workflowInstanceKey", job.getProcessInstanceKey()); + sendToCamelRoute(RouteId.ACCOUNT_LOOKUP_CALLBACK, exchange); + client.newCompleteCommand(job.getKey()).variables(variables).send(); + })); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupWorker.java new file mode 100644 index 00000000..4fbea690 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupWorker.java @@ -0,0 +1,68 @@ +package org.mifos.processor.bulk.zeebe.worker; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.CACHED_TRANSACTION_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.REGISTERING_INSTITUTE_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; +import static org.mifos.processor.bulk.zeebe.worker.Worker.BATCH_ACCOUNT_LOOKUP; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.zeebe.client.ZeebeClient; +import java.util.Map; +import org.apache.camel.CamelContext; +import org.apache.camel.Exchange; +import org.apache.camel.ProducerTemplate; +import org.apache.camel.support.DefaultExchange; +import org.mifos.processor.bulk.camel.routes.RouteId; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class BatchAccountLookupWorker extends BaseWorker { + + @Autowired + private ZeebeClient zeebeClient; + + @Autowired + private ProducerTemplate producerTemplate; + @Autowired + private CamelContext camelContext; + @Autowired + private ObjectMapper objectMapper; + @Value("${identity_account_mapper.hostname}") + private String identityMapperURL; + @Value("${bulk-processor.hostname}") + private String bulkURL; + @Value("${identity_account_mapper.batch_account_lookup_callback}") + private String batchAccountLookupCallback; + + @Override + public void setup() { + + newWorker(BATCH_ACCOUNT_LOOKUP, ((client, job) -> { + logger.info("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); + Map variables = job.getVariablesAsMap(); + Exchange exchange = new DefaultExchange(camelContext); + String filename = (String) variables.get(FILE_NAME); + String registeringInstituteId = variables.get(REGISTERING_INSTITUTE_ID).toString(); + logger.info("registeringInstituteId in worker {}", registeringInstituteId); + variables.put(CACHED_TRANSACTION_ID, job.getKey()); + exchange.setProperty(HEADER_REGISTERING_INSTITUTE_ID, registeringInstituteId) ; + exchange.setProperty(SERVER_FILE_NAME, filename); + exchange.setProperty(REQUEST_ID, job.getKey()); + exchange.setProperty(CALLBACK, identityMapperURL + batchAccountLookupCallback); + + try { + sendToCamelRoute(RouteId.ACCOUNT_LOOKUP, exchange); + } catch (Exception e) { + variables.put(PARTY_LOOKUP_FAILED, true); + } + client.newCompleteCommand(job.getKey()).variables(variables).send(); + })); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java index 50a52251..8ba21146 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java @@ -8,6 +8,7 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_FAILURE_SUB_BATCHES; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_SUCCESS_SUB_BATCHES; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_FAILED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_FAILED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SUB_BATCHES; @@ -39,9 +40,11 @@ public void setup() { } String filename = (String) variables.get(FILE_NAME); + Boolean partyLookupFailed = (Boolean) variables.get(PARTY_LOOKUP_FAILED); Exchange exchange = new DefaultExchange(camelContext); exchange.setProperty(SERVER_FILE_NAME, filename); exchange.setProperty(ZEEBE_VARIABLE, variables); + exchange.setProperty("partyLookupFailed", partyLookupFailed); exchange.setProperty(SUB_BATCH_DETAILS, new ArrayList()); try { diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java index 49610a69..f508dca5 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/Worker.java @@ -4,8 +4,9 @@ public enum Worker { PARTY_LOOKUP("partyLookup"), APPROVAL("approval"), ORDERING("ordering"), SPLITTING("splitting"), FORMATTING("formatting"), BATCH_STATUS( "batchStatus"), SEND_CALLBACK("sendCallback"), MERGE_BACK("mergeSubBatch"), INIT_SUB_BATCH("initSubBatch"), ACCOUNT_LOOKUP( - "accountLookup"), ACCOUNT_LOOKUP_CALLBACK("accountLookupCallback"), BATCH_AGGREGATE( - "batchAggregate"), AUTHORIZATION("authorization"), DE_DEPLICATION("deduplicate"); + "accountLookup"), ACCOUNT_LOOKUP_CALLBACK("accountLookupCallback"), BATCH_AGGREGATE("batchAggregate"), AUTHORIZATION( + "authorization"), DE_DEPLICATION("deduplicate"), BATCH_ACCOUNT_LOOKUP( + "batchAccountLookup"), BATCH_ACCOUNT_LOOKUP_CALLBACK("batchAccountLookupCallback"); private final String value; diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 7e62af70..6cdd71a3 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -150,6 +150,8 @@ identity_account_mapper: hostname : "http://ph-ee-identity-account-mapper:80" account_lookup: /beneficiary account_lookup_callback: /accountLookupCallback + batch_account_lookup: /accountLookup + batch_account_lookup_callback: /batchAccountLookupCallback bulk_processor: hostname : "https://ph-ee-connector-bulk:8443" @@ -213,3 +215,16 @@ security-server: gov-stack-client: header-key: "X-GovStack-Client" header-value: "PAYMENT-BB" + + +bpmns: + tenants: + - id: "gorilla" + flows: + batch-transactions: "bulk_processor-{dfspid}" + - id: "rhino" + flows: + batch-transactions: "bulk_processor_account_lookup-{dfspid}" + - id: "lion" + flows: + batch-transactions: "bulk_processor-{dfspid}" From 8d2fb3002950842910089dc2f1dfd37deea302ae Mon Sep 17 00:00:00 2001 From: Dhaval Maniyar <100835289+logoutdhaval@users.noreply.github.com> Date: Tue, 2 Jan 2024 20:30:50 +0530 Subject: [PATCH 112/156] [PHEE-522] Create Sync callback functionality in bulk (#137) * callback if check * readable logs --- .../processor/bulk/camel/routes/SendCallbackRoute.java | 7 ++++--- .../bulk/zeebe/worker/BatchAccountLookupWorker.java | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java index 546a3cce..2244e448 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java @@ -44,10 +44,11 @@ public void configure() throws Exception { ObjectMapper objectMapper = new ObjectMapper(); String jsonString = objectMapper.writeValueAsString(body); exchange.getIn().setBody(jsonString); - }).setHeader(Exchange.HTTP_METHOD, constant("POST")) + }).choice().when(exchangeProperty("X-CallbackURL").isNotNull()).setHeader(Exchange.HTTP_METHOD, constant("POST")) .toD("${exchangeProperty.X-CallbackURL}?bridgeEndpoint=true&throwExceptionOnFailure=false") - .log(LoggingLevel.INFO, "Callback Response body: ${body}").choice() - .when(header(Exchange.HTTP_RESPONSE_CODE).regex("^2\\d{2}$")).log(LoggingLevel.INFO, "Callback sending was successful") + .log(LoggingLevel.INFO, "Callback Response body: ${body}").endChoice().otherwise() + .log("Unable to send callback: callback url is null").choice().when(header(Exchange.HTTP_RESPONSE_CODE).regex("^2\\d{2}$")) + .when(exchangeProperty("X-CallbackURL").isNotNull()).log(LoggingLevel.INFO, "Callback sending was successful") .process(exchange -> { List phases = exchange.getProperty(PHASES, List.class); exchange.setProperty(CALLBACK_RESPONSE_CODE, exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE)); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupWorker.java index 4fbea690..a4861ba4 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupWorker.java @@ -52,7 +52,7 @@ public void setup() { String registeringInstituteId = variables.get(REGISTERING_INSTITUTE_ID).toString(); logger.info("registeringInstituteId in worker {}", registeringInstituteId); variables.put(CACHED_TRANSACTION_ID, job.getKey()); - exchange.setProperty(HEADER_REGISTERING_INSTITUTE_ID, registeringInstituteId) ; + exchange.setProperty(HEADER_REGISTERING_INSTITUTE_ID, registeringInstituteId); exchange.setProperty(SERVER_FILE_NAME, filename); exchange.setProperty(REQUEST_ID, job.getKey()); exchange.setProperty(CALLBACK, identityMapperURL + batchAccountLookupCallback); From 7962ba647f391f75687289a9e7b24818e64174c2 Mon Sep 17 00:00:00 2001 From: Apurb Rajdhan <103931815+apurbraj@users.noreply.github.com> Date: Tue, 2 Jan 2024 22:56:56 +0530 Subject: [PATCH 113/156] Add colum in yaml file (#138) --- src/main/resources/application.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 6cdd71a3..afc84419 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -157,7 +157,7 @@ bulk_processor: hostname : "https://ph-ee-connector-bulk:8443" csv: - columnNames: "id,request_id,payment_mode,payer_identifier_type,payer_identifier,payee_identifier_type,payee_identifier,amount,currency,note,account_number,program_shortcode,cycle" + columnNames: "id,request_id,payment_mode,payer_identifier_type,payer_identifier,payee_identifier_type,payee_identifier,amount,currency,note,account_number,program_shortcode,cycle,payee_dfsp_id" size : 100000 # in bytes budget-account: From c0951140c7d943a3f5e0618555c9c82559fefae9 Mon Sep 17 00:00:00 2001 From: Sourabh Kumar <148945054+shkr7@users.noreply.github.com> Date: Wed, 17 Jan 2024 17:58:46 +0530 Subject: [PATCH 114/156] [PHEE-533] Fix for BPMN 500 issue (#141) * Fixed BPMN 500 issue * Fixed checkstyle issue * Fixed checkstyle issue --------- Co-authored-by: Sourabh --- .github/pull_request_template.md | 2 +- .../BatchTransactionsController.java | 10 ++++++++ .../camel/routes/ProcessorStartRoute.java | 11 +++++--- .../bulk/schema/ExceptionMapperDTO.java | 18 +++++++++++++ .../GlobalExceptionMapper.java | 25 +++++++++++++++++++ 5 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/schema/ExceptionMapperDTO.java create mode 100644 src/main/java/org/mifos/processor/exceptionmapper/GlobalExceptionMapper.java diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 98a182a2..bcf84d8d 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -11,7 +11,7 @@ ex: [phee-123] PR title. Please make sure these boxes are checked before submitting your pull request - thanks! - [ ] Followed the PR title naming convention mentioned above. -- [ ] Design related bullet points or design document link related to this PR added in the description above. +- [ ] Design related bullet points or design document link related to this PR added in the description above. - [ ] Updated corresponding Postman Collection or Api documentation for the changes in this PR. diff --git a/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java b/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java index 38129131..90df1c72 100644 --- a/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java +++ b/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java @@ -12,6 +12,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import io.camunda.zeebe.client.api.command.ClientStatusException; +import io.grpc.Status; import java.nio.charset.Charset; import java.util.List; import java.util.Optional; @@ -115,6 +117,7 @@ public String handleMultipartException(HttpServletResponse httpServletResponse) private CamelApiResponse sendRequestToCamel(Headers headers) { Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(), headers); exchange = producerTemplate.send("direct:post-batch-transactions", exchange); + checkAndThrowClientStatusException(exchange); int statusCode = exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE, Integer.class); String body = exchange.getIn().getBody(String.class); return new CamelApiResponse(body, statusCode); @@ -151,4 +154,11 @@ private Optional isValidRequest(HttpServletRequest httpServletRequest, S } return response; } + + private void checkAndThrowClientStatusException(Exchange exchange) { + Exception cause = exchange.getProperty(Exchange.EXCEPTION_CAUGHT, Exception.class); + if (cause instanceof ClientStatusException) { + throw new ClientStatusException(Status.FAILED_PRECONDITION, cause); + } + } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 0a508899..8b7e4554 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -46,6 +46,7 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.THRESHOLD_DELAY; import com.fasterxml.jackson.core.JsonProcessingException; +import io.camunda.zeebe.client.api.command.ClientStatusException; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -146,8 +147,7 @@ private void setup() { exchange.setProperty(BATCH_ID, batchId); }).to("direct:validateFileSyncResponse").choice().when(header("CamelHttpResponseCode").isNotEqualTo("200")) - .log(LoggingLevel.ERROR, "File upload failed").otherwise().wireTap("direct:executeBatch").to("direct:pollingOutput") - .endChoice().endChoice(); + .log(LoggingLevel.ERROR, "File upload failed").otherwise().to("direct:executeBatch").endChoice().endChoice(); from("direct:post-bulk-transfer").unmarshal().mimeMultipart("multipart/*").to("direct:validate-tenant").process(exchange -> { String fileName = System.currentTimeMillis() + "_" + exchange.getIn().getHeader("fileName", String.class); @@ -160,7 +160,7 @@ private void setup() { exchange.setProperty(FILE_NAME, fileName); exchange.setProperty(REQUEST_ID, requestId); exchange.setProperty(PURPOSE, purpose); - }).wireTap("direct:start-batch-process-csv").to("direct:pollingOutput"); + }).wireTap("direct:start-batch-process-csv"); from("direct:validate-tenant").id("direct:validate-tenant").log("Validating tenant").process(exchange -> { String tenantName = exchange.getIn().getHeader(HEADER_PLATFORM_TENANT_ID, String.class); @@ -306,6 +306,9 @@ private void setup() { response.put("request_id", requestId); response.put("status", "queued"); } + } catch (ClientStatusException c) { + log.error("Got ClientStatusException : {}", c.getMessage()); + throw c; } catch (Exception e) { response.put("errorCode", 500); response.put("errorDescription", "Unable to start zeebe workflow"); @@ -314,7 +317,7 @@ private void setup() { exchange.getIn().setBody(response.toString()); - }).log("Completed route direct:start-batch-process-csv"); + }).log("Completed route direct:start-batch-process-csv").to("direct:pollingOutput"); from("direct:start-batch-process-raw").id("direct:start-batch-process-raw").log("Starting route direct:start-batch-process-raw") .process(exchange -> { diff --git a/src/main/java/org/mifos/processor/bulk/schema/ExceptionMapperDTO.java b/src/main/java/org/mifos/processor/bulk/schema/ExceptionMapperDTO.java new file mode 100644 index 00000000..8b0ad44f --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/ExceptionMapperDTO.java @@ -0,0 +1,18 @@ +package org.mifos.processor.bulk.schema; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +@Getter +@Setter +@ToString +@NoArgsConstructor +@AllArgsConstructor +public class ExceptionMapperDTO { + + private String responseCode; + private String responseDescription; +} diff --git a/src/main/java/org/mifos/processor/exceptionmapper/GlobalExceptionMapper.java b/src/main/java/org/mifos/processor/exceptionmapper/GlobalExceptionMapper.java new file mode 100644 index 00000000..5b35c288 --- /dev/null +++ b/src/main/java/org/mifos/processor/exceptionmapper/GlobalExceptionMapper.java @@ -0,0 +1,25 @@ +package org.mifos.processor.exceptionmapper; + +import io.camunda.zeebe.client.api.command.ClientStatusException; +import org.mifos.processor.bulk.schema.ExceptionMapperDTO; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +@RestControllerAdvice +public class GlobalExceptionMapper { + + @ExceptionHandler(ClientStatusException.class) + public ResponseEntity handleClientStatusException(ClientStatusException ex) { + ExceptionMapperDTO dto = new ExceptionMapperDTO("01", "Process definition not found"); + return ResponseEntity.status(HttpStatus.PRECONDITION_FAILED).contentType(MediaType.APPLICATION_JSON).body(dto); + } + + @ExceptionHandler(Exception.class) + public ResponseEntity handleException(Exception ex) { + ExceptionMapperDTO dto = new ExceptionMapperDTO("01", ex.getMessage()); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).contentType(MediaType.APPLICATION_JSON).body(dto); + } +} From 162dc0e8e08e28a1a732d18da5c22a4761c12632 Mon Sep 17 00:00:00 2001 From: Dhaval Maniyar <100835289+logoutdhaval@users.noreply.github.com> Date: Wed, 17 Jan 2024 20:44:44 +0530 Subject: [PATCH 115/156] [PHEE-535] Extract content type boundary logic fix. (#142) * [PHEE-535] Extract content type boundary logic fix. * change in version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a11df0f2..8804e2a4 100644 --- a/build.gradle +++ b/build.gradle @@ -80,7 +80,7 @@ dependencies { // miscellaneous dependency implementation 'com.google.code.gson:gson:2.8.9' implementation 'org.json:json:20210307' - implementation 'org.mifos:ph-ee-connector-common:1.7.0-SNAPSHOT' + implementation 'org.mifos:ph-ee-connector-common:1.8.1-SNAPSHOT' implementation 'org.apache.camel.springboot:camel-spring-boot-starter:3.4.0' implementation 'org.apache.camel:camel-undertow:3.4.0' implementation 'org.springframework.boot:spring-boot-starter:2.5.2' From 18be6b655ca6320977145b09774dec6a9692559c Mon Sep 17 00:00:00 2001 From: Sourabh Kumar <148945054+shkr7@users.noreply.github.com> Date: Mon, 12 Feb 2024 18:37:17 +0530 Subject: [PATCH 116/156] [PHEE-547] Convert batch-account-lookup camel route to Java/Spring (#143) * [PHEE-547] Convert batch-account-lookup camel route to Java/Spring * Added checkstyle changes * Added genric Retrofit builder and API calls * Added spotless change --------- Co-authored-by: Sourabh --- build.gradle | 5 ++ .../bulk/camel/routes/AccountLookupRoute.java | 6 +- .../bulk/connectors/api/AccountLookupApi.java | 17 +++++ .../service/AccountLookupService.java | 44 +++++++++++++ .../connectors/service/RetrofitService.java | 13 ++++ .../bulk/service/BatchAccountLookup.java | 63 +++++++++++++++++++ .../mifos/processor/bulk/utility/Headers.java | 8 +++ 7 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/connectors/api/AccountLookupApi.java create mode 100644 src/main/java/org/mifos/processor/bulk/connectors/service/AccountLookupService.java create mode 100644 src/main/java/org/mifos/processor/bulk/connectors/service/RetrofitService.java create mode 100644 src/main/java/org/mifos/processor/bulk/service/BatchAccountLookup.java diff --git a/build.gradle b/build.gradle index 8804e2a4..8cb4ef4b 100644 --- a/build.gradle +++ b/build.gradle @@ -109,6 +109,11 @@ dependencies { testImplementation "com.google.truth:truth:1.1.3" testImplementation 'com.google.code.gson:gson:2.9.0' implementation 'io.rest-assured:rest-assured:4.4.0' + + //retrofit + implementation 'com.squareup.retrofit2:retrofit:2.9.0' + implementation 'com.squareup.retrofit2:converter-jackson:2.9.0' + implementation('com.squareup.retrofit2:converter-gson:2.4.0') } configure(this) { diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java index 2b5c55c3..57fcef9a 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java @@ -19,6 +19,7 @@ import org.mifos.connector.common.identityaccountmapper.dto.AccountMapperRequestDTO; import org.mifos.connector.common.identityaccountmapper.dto.BeneficiaryDTO; import org.mifos.processor.bulk.schema.Transaction; +import org.mifos.processor.bulk.service.BatchAccountLookup; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -58,8 +59,9 @@ public void process(Exchange exchange) throws Exception { from(RouteId.ACCOUNT_LOOKUP.getValue()).id(RouteId.ACCOUNT_LOOKUP.getValue()).log("Starting route " + RouteId.ACCOUNT_LOOKUP.name()) .process(exchange -> exchange.setProperty(OVERRIDE_HEADER, true)).to("direct:download-file") - .to("direct:get-transaction-array").to("direct:batch-account-lookup").to("direct:update-file").to("direct:upload-file") - .process(exchange -> { + .to("direct:get-transaction-array").bean(BatchAccountLookup.class, "doBatchAccountLookup") + // .to("direct:batch-account-lookup") + .to("direct:update-file").to("direct:upload-file").process(exchange -> { exchange.setProperty(PARTY_LOOKUP_FAILED, false); }); diff --git a/src/main/java/org/mifos/processor/bulk/connectors/api/AccountLookupApi.java b/src/main/java/org/mifos/processor/bulk/connectors/api/AccountLookupApi.java new file mode 100644 index 00000000..70985acb --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/connectors/api/AccountLookupApi.java @@ -0,0 +1,17 @@ +package org.mifos.processor.bulk.connectors.api; + +import java.util.Map; +import org.mifos.connector.common.identityaccountmapper.dto.AccountMapperRequestDTO; +import org.springframework.stereotype.Component; +import retrofit2.Call; +import retrofit2.http.Body; +import retrofit2.http.HeaderMap; +import retrofit2.http.POST; +import retrofit2.http.Url; + +@Component +public interface AccountLookupApi { + + @POST + Call batchAccountLookup(@Url String fullUrl, @Body AccountMapperRequestDTO requestBody, @HeaderMap Map headers); +} diff --git a/src/main/java/org/mifos/processor/bulk/connectors/service/AccountLookupService.java b/src/main/java/org/mifos/processor/bulk/connectors/service/AccountLookupService.java new file mode 100644 index 00000000..aba71029 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/connectors/service/AccountLookupService.java @@ -0,0 +1,44 @@ +package org.mifos.processor.bulk.connectors.service; + +import java.io.IOException; +import java.util.Map; +import org.mifos.connector.common.identityaccountmapper.dto.AccountMapperRequestDTO; +import org.mifos.processor.bulk.connectors.api.AccountLookupApi; +import org.mifos.processor.bulk.utility.Headers; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import retrofit2.Call; +import retrofit2.Retrofit; + +@Service +public class AccountLookupService { + + @Autowired + RetrofitService retrofitService; + + public Logger logger = LoggerFactory.getLogger(this.getClass()); + + public void accountLookupCall(String baseUrl, String fullUrl, AccountMapperRequestDTO requestBody, Map headers) + throws IOException { + Retrofit retrofit = retrofitService.createRetrofit(baseUrl); + + AccountLookupApi accountLookupApi = retrofit.create(AccountLookupApi.class); + + Call call = accountLookupApi.batchAccountLookup(fullUrl, requestBody, Headers.convertHeaders(headers)); + + try { + retrofit2.Response response = call.execute(); + if (response.isSuccessful()) { + Object apiResponse = response.body(); + logger.debug("API response is :: {}", apiResponse); + } else { + logger.error("Error occurred. HTTP status code: {}", response.code()); + } + } catch (IOException e) { + logger.error("Error making Retrofit API call", e); + throw e; + } + } +} diff --git a/src/main/java/org/mifos/processor/bulk/connectors/service/RetrofitService.java b/src/main/java/org/mifos/processor/bulk/connectors/service/RetrofitService.java new file mode 100644 index 00000000..8dc80da5 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/connectors/service/RetrofitService.java @@ -0,0 +1,13 @@ +package org.mifos.processor.bulk.connectors.service; + +import org.springframework.stereotype.Component; +import retrofit2.Retrofit; +import retrofit2.converter.gson.GsonConverterFactory; + +@Component +public class RetrofitService { + + public Retrofit createRetrofit(String baseUrl) { + return new Retrofit.Builder().baseUrl(baseUrl).addConverterFactory(GsonConverterFactory.create()).build(); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/service/BatchAccountLookup.java b/src/main/java/org/mifos/processor/bulk/service/BatchAccountLookup.java new file mode 100644 index 00000000..77ea2f99 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/service/BatchAccountLookup.java @@ -0,0 +1,63 @@ +package org.mifos.processor.bulk.service; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REGISTERING_INSTITUTION_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.camel.Exchange; +import org.mifos.connector.common.identityaccountmapper.dto.AccountMapperRequestDTO; +import org.mifos.connector.common.identityaccountmapper.dto.BeneficiaryDTO; +import org.mifos.processor.bulk.connectors.service.AccountLookupService; +import org.mifos.processor.bulk.schema.Transaction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; + +public class BatchAccountLookup { + + @Autowired + private ObjectMapper objectMapper; + @Autowired + private AccountLookupService accountLookupService; + @Value("${identity_account_mapper.hostname}") + private String identityEndpoint; + @Value("${identity_account_mapper.batch_account_lookup}") + private String batchAccountLookup; + + public Logger logger = LoggerFactory.getLogger(this.getClass()); + + public void doBatchAccountLookup(Exchange exchange) throws IOException { + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + HashMap> stringListHashMap = new HashMap<>(); + List beneficiaryDTOList = new ArrayList<>(); + transactionList.forEach(transaction -> { + beneficiaryDTOList.add(new BeneficiaryDTO(transaction.getPayeeIdentifier(), "", "", "")); + }); + String requestId = exchange.getProperty(REQUEST_ID, String.class); + String callbackUrl = exchange.getProperty(CALLBACK, String.class); + String registeringInstitutionId = exchange.getProperty(HEADER_REGISTERING_INSTITUTE_ID, String.class); + AccountMapperRequestDTO accountMapperRequestDTO = new AccountMapperRequestDTO(requestId, registeringInstitutionId, + beneficiaryDTOList); + String requestBody = objectMapper.writeValueAsString(accountMapperRequestDTO); + + exchange.getIn().setHeader(CALLBACK, callbackUrl); + exchange.getIn().setHeader(REGISTERING_INSTITUTION_ID, registeringInstitutionId); + exchange.getIn().setHeader("Content-type", "application/json"); + exchange.getIn().setBody(requestBody); + + Map headers = new HashMap<>(); + headers = exchange.getIn().getHeaders(); + + String fullUrl = identityEndpoint + batchAccountLookup; + accountLookupService.accountLookupCall(identityEndpoint, fullUrl, accountMapperRequestDTO, headers); + } +} diff --git a/src/main/java/org/mifos/processor/bulk/utility/Headers.java b/src/main/java/org/mifos/processor/bulk/utility/Headers.java index 03bff690..f46aba91 100644 --- a/src/main/java/org/mifos/processor/bulk/utility/Headers.java +++ b/src/main/java/org/mifos/processor/bulk/utility/Headers.java @@ -26,6 +26,14 @@ public Object get(String key) { return this.headers.get(key); } + public static Map convertHeaders(Map headers) { + Map stringHeaders = new HashMap<>(); + for (Map.Entry entry : headers.entrySet()) { + stringHeaders.put(entry.getKey(), entry.getValue().toString()); + } + return stringHeaders; + } + public static class HeaderBuilder { private Map headers = new HashMap<>(); From af332b38973702004ae9e102742c6417113294e3 Mon Sep 17 00:00:00 2001 From: Sourabh Kumar <148945054+shkr7@users.noreply.github.com> Date: Tue, 13 Feb 2024 19:23:28 +0530 Subject: [PATCH 117/156] [PHEE-578] Convert FileRoute to Java/Spring (#146) * Added Java fileRoute * Added FileRouteService --------- Co-authored-by: Sourabh --- .../bulk/camel/routes/AccountLookupRoute.java | 8 ++-- .../bulk/service/FileRouteService.java | 48 +++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/service/FileRouteService.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java index 57fcef9a..b14bee5b 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java @@ -20,6 +20,7 @@ import org.mifos.connector.common.identityaccountmapper.dto.BeneficiaryDTO; import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.service.BatchAccountLookup; +import org.mifos.processor.bulk.service.FileRouteService; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -58,10 +59,9 @@ public void process(Exchange exchange) throws Exception { .log("API Response: ${body}").process(disableSslProcessor); from(RouteId.ACCOUNT_LOOKUP.getValue()).id(RouteId.ACCOUNT_LOOKUP.getValue()).log("Starting route " + RouteId.ACCOUNT_LOOKUP.name()) - .process(exchange -> exchange.setProperty(OVERRIDE_HEADER, true)).to("direct:download-file") - .to("direct:get-transaction-array").bean(BatchAccountLookup.class, "doBatchAccountLookup") - // .to("direct:batch-account-lookup") - .to("direct:update-file").to("direct:upload-file").process(exchange -> { + .process(exchange -> exchange.setProperty(OVERRIDE_HEADER, true)).bean(FileRouteService.class, "downloadFile") + .to("direct:get-transaction-array").bean(BatchAccountLookup.class, "doBatchAccountLookup").to("direct:update-file") + .bean(FileRouteService.class, "uploadFile").process(exchange -> { exchange.setProperty(PARTY_LOOKUP_FAILED, false); }); diff --git a/src/main/java/org/mifos/processor/bulk/service/FileRouteService.java b/src/main/java/org/mifos/processor/bulk/service/FileRouteService.java new file mode 100644 index 00000000..ec50c22b --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/service/FileRouteService.java @@ -0,0 +1,48 @@ +package org.mifos.processor.bulk.service; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import lombok.extern.slf4j.Slf4j; +import org.apache.camel.Exchange; +import org.mifos.processor.bulk.file.FileTransferService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +@Service +@Slf4j +public class FileRouteService { + + @Autowired + @Qualifier("awsStorage") + private FileTransferService fileTransferService; + @Value("${application.bucket-name}") + private String bucketName; + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + public void downloadFile(Exchange exchange) throws IOException { + String filename = exchange.getProperty(SERVER_FILE_NAME, String.class); + + byte[] csvFile = fileTransferService.downloadFile(filename, bucketName); + File file = new File(filename); + try (FileOutputStream fos = new FileOutputStream(file)) { + fos.write(csvFile); + } + exchange.setProperty(LOCAL_FILE_PATH, file.getAbsolutePath()); + logger.info("File downloaded"); + } + + public void uploadFile(Exchange exchange) { + String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); + String serverFileName = fileTransferService.uploadFile(new File(filepath), bucketName); + exchange.setProperty(SERVER_FILE_NAME, serverFileName); + logger.info("Uploaded file: {}", serverFileName); + } +} From 42e5aa0401199e36b34e67107f203044362e07ac Mon Sep 17 00:00:00 2001 From: Anover000 <140409886+Anover000@users.noreply.github.com> Date: Thu, 15 Feb 2024 18:32:18 +0530 Subject: [PATCH 118/156] [PHEE-560] Reduce Memory Consumption of reported services (#147) * upgrade zeebe version * increasing poll Interval * zeebe max-ex-trds 1000->50 * make zeebe pollInterval configurable --- build.gradle | 2 +- .../mifos/processor/bulk/zeebe/ZeebeClientConfiguration.java | 5 ++++- src/main/resources/application.yaml | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 8cb4ef4b..29d95e1b 100644 --- a/build.gradle +++ b/build.gradle @@ -95,7 +95,7 @@ dependencies { implementation 'com.amazonaws:aws-java-sdk-dynamodb:1.11.486' implementation 'com.azure:azure-storage-blob:12.12.0' - implementation 'io.camunda:zeebe-client-java:8.1.1' + implementation 'io.camunda:zeebe-client-java:8.1.23' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.12.3' implementation 'org.apache.tika:tika-core:1.4' implementation 'org.apache.commons:commons-io:1.3.2' diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeClientConfiguration.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeClientConfiguration.java index 5afd9130..585c5b5c 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeClientConfiguration.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeClientConfiguration.java @@ -15,10 +15,13 @@ public class ZeebeClientConfiguration { @Value("${zeebe.client.max-execution-threads}") private int zeebeClientMaxThreads; + @Value("${zeebe.client.poll-interval}") + private int zeebeClientPollInterval; + @Bean public ZeebeClient setup() { return ZeebeClient.newClientBuilder().gatewayAddress(zeebeBrokerContactpoint).usePlaintext() - .defaultJobPollInterval(Duration.ofMillis(1)).defaultJobWorkerMaxJobsActive(2000) + .defaultJobPollInterval(Duration.ofMillis(zeebeClientPollInterval)).defaultJobWorkerMaxJobsActive(2000) .numJobWorkerExecutionThreads(zeebeClientMaxThreads).build(); } } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index afc84419..3da8dfdd 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -20,8 +20,9 @@ application: zeebe: client: - max-execution-threads: 1000 + max-execution-threads: 50 evenly-allocated-max-jobs: 1000 + poll-interval: 10 # max-execution-threads: 100 # number-of-workers: 8 # evenly-allocated-max-jobs: "#{${zeebe.client.max-execution-threads} / ${zeebe.client.number-of-workers}}" From c96c0fd7416c46924e1874440043cd089d0e2c02 Mon Sep 17 00:00:00 2001 From: Sourabh Kumar <148945054+shkr7@users.noreply.github.com> Date: Fri, 16 Feb 2024 17:55:30 +0530 Subject: [PATCH 119/156] [PHEE-581] Fixed 404 issue in aggregate worker (#148) * Modified aggregate worker * Increased retry-count --- .../processor/bulk/zeebe/worker/AggregateWorker.java | 8 +++++++- src/main/resources/application.yaml | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java index 3c30f9fc..66272d44 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java @@ -46,7 +46,13 @@ public void setup() { Boolean batchStatusFailed = exchange.getProperty(BATCH_STATUS_FAILED, Boolean.class); if (batchStatusFailed == null || !batchStatusFailed) { - successRate = exchange.getProperty(COMPLETION_RATE, Long.class).intValue(); + if (exchange.getException() != null && exchange.getException().getMessage() != null + && exchange.getException().getMessage().contains("404")) { + logger.error("An error occurred, retrying"); + successRate = 0; + } else { + successRate = exchange.getProperty(COMPLETION_RATE, Long.class).intValue(); + } } else { variables.put(ERROR_CODE, exchange.getProperty(ERROR_CODE)); variables.put(ERROR_DESCRIPTION, exchange.getProperty(ERROR_DESCRIPTION)); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 3da8dfdd..257dd962 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -111,7 +111,7 @@ config: completion-threshold-check: enable: false completion-threshold: 95 # in percentage - max-retry: 4 #can be as high as 30 + max-retry: 15 #can be as high as 30 delay: 2 # in seconds deduplication: enabled: true From 87d542a67f0b6e24f2f15e69dbe8b83c14c29876 Mon Sep 17 00:00:00 2001 From: Sourabh Kumar <148945054+shkr7@users.noreply.github.com> Date: Wed, 21 Feb 2024 12:34:22 +0530 Subject: [PATCH 120/156] [PHEE-582] FileProcessingRoute to Java/Spring in bulk (#149) * Added java File processing routes * Converted FileProcessingRoute to Java/Spring --------- Co-authored-by: Sourabh --- .../bulk/camel/routes/AccountLookupRoute.java | 5 +- .../service/FileProcessingRouteService.java | 90 +++++++++++++++++++ 2 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/service/FileProcessingRouteService.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java index b14bee5b..e40a0c48 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java @@ -20,6 +20,7 @@ import org.mifos.connector.common.identityaccountmapper.dto.BeneficiaryDTO; import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.service.BatchAccountLookup; +import org.mifos.processor.bulk.service.FileProcessingRouteService; import org.mifos.processor.bulk.service.FileRouteService; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -60,8 +61,8 @@ public void process(Exchange exchange) throws Exception { from(RouteId.ACCOUNT_LOOKUP.getValue()).id(RouteId.ACCOUNT_LOOKUP.getValue()).log("Starting route " + RouteId.ACCOUNT_LOOKUP.name()) .process(exchange -> exchange.setProperty(OVERRIDE_HEADER, true)).bean(FileRouteService.class, "downloadFile") - .to("direct:get-transaction-array").bean(BatchAccountLookup.class, "doBatchAccountLookup").to("direct:update-file") - .bean(FileRouteService.class, "uploadFile").process(exchange -> { + .bean(FileProcessingRouteService.class, "getTxnArray").bean(BatchAccountLookup.class, "doBatchAccountLookup") + .bean(FileProcessingRouteService.class, "updateFile").bean(FileRouteService.class, "uploadFile").process(exchange -> { exchange.setProperty(PARTY_LOOKUP_FAILED, false); }); diff --git a/src/main/java/org/mifos/processor/bulk/service/FileProcessingRouteService.java b/src/main/java/org/mifos/processor/bulk/service/FileProcessingRouteService.java new file mode 100644 index 00000000..4a803920 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/service/FileProcessingRouteService.java @@ -0,0 +1,90 @@ +package org.mifos.processor.bulk.service; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; +import static org.mifos.processor.bulk.camel.config.CamelProperties.OVERRIDE_HEADER; +import static org.mifos.processor.bulk.camel.config.CamelProperties.RESULT_TRANSACTION_LIST; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_LENGTH; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETED_AMOUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FAILED_AMOUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ONGOING_AMOUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TOTAL_AMOUNT; + +import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.databind.SequenceWriter; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.apache.camel.Exchange; +import org.mifos.processor.bulk.schema.Transaction; +import org.mifos.processor.bulk.schema.TransactionResult; +import org.mifos.processor.bulk.utility.CsvWriter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +@Slf4j +public class FileProcessingRouteService { + + @Autowired + private CsvMapper csvMapper; + + public void getTxnArray(Exchange exchange) throws IOException { + Double totalAmount = 0.0; + Long failedAmount = 0L; + Long completedAmount = 0L; + String filename = exchange.getProperty(LOCAL_FILE_PATH, String.class); + log.debug("Local file path: {}", filename); + CsvSchema schema = CsvSchema.emptySchema().withHeader(); + log.info("Filename: {}", filename); + FileReader reader = new FileReader(filename); + MappingIterator readValues = csvMapper.readerWithSchemaFor(Transaction.class).with(schema).readValues(reader); + List transactionList = new ArrayList<>(); + while (readValues.hasNext()) { + Transaction current = readValues.next(); + transactionList.add(current); + totalAmount += Double.parseDouble(current.getAmount()); + } + reader.close(); + exchange.setProperty(TRANSACTION_LIST, transactionList); + exchange.setProperty(TRANSACTION_LIST_LENGTH, transactionList.size()); + exchange.setProperty(TOTAL_AMOUNT, totalAmount); + exchange.setProperty(ONGOING_AMOUNT, totalAmount); + exchange.setProperty(FAILED_AMOUNT, failedAmount); + exchange.setProperty(COMPLETED_AMOUNT, completedAmount); + } + + public void updateResultFile(Exchange exchange) throws IOException { + String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); + List transactionList = exchange.getProperty(RESULT_TRANSACTION_LIST, List.class); + + Boolean overrideHeader = exchange.getProperty(OVERRIDE_HEADER, Boolean.class); + + CsvWriter.writeToCsv(transactionList, TransactionResult.class, csvMapper, overrideHeader, filepath); + } + + public void updateFile(Exchange exchange) throws IOException { + String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + + // getting header + Boolean overrideHeader = exchange.getProperty(OVERRIDE_HEADER, Boolean.class); + CsvSchema csvSchema = csvMapper.schemaFor(Transaction.class); + if (overrideHeader) { + csvSchema = csvSchema.withHeader(); + } else { + csvSchema = csvSchema.withoutHeader(); + } + + File file = new File(filepath); + SequenceWriter writer = csvMapper.writerWithSchemaFor(Transaction.class).with(csvSchema).writeValues(file); + for (Transaction transaction : transactionList) { + writer.write(transaction); + } + } +} From 5fbd55e15e0d0d834974b24717a553aee6474ffc Mon Sep 17 00:00:00 2001 From: Anover000 <140409886+Anover000@users.noreply.github.com> Date: Wed, 21 Feb 2024 23:38:58 +0530 Subject: [PATCH 121/156] added bulk processor flow to wakanda (#153) --- src/main/resources/application.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 257dd962..ead7b933 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -223,6 +223,9 @@ bpmns: - id: "gorilla" flows: batch-transactions: "bulk_processor-{dfspid}" + - id: "wakanda" + flows: + batch-transactions: "bulk_processor-{dfspid}" - id: "rhino" flows: batch-transactions: "bulk_processor_account_lookup-{dfspid}" From 03a9566f455839c1631237ba52ab5f298e1dbd41 Mon Sep 17 00:00:00 2001 From: Sourabh Kumar <148945054+shkr7@users.noreply.github.com> Date: Thu, 22 Feb 2024 15:24:47 +0530 Subject: [PATCH 122/156] [PHEE-586] Convert BD-003 camel routes to Java/Spring (#150) * Added BD-003 routes * Added spotless change * Added config for polling API --------- Co-authored-by: Sourabh --- .../bulk/camel/config/CamelProperties.java | 1 + .../camel/routes/ProcessorStartRoute.java | 18 ++--- .../service/ProcessorStartRouteService.java | 71 +++++++++++++++++++ src/main/resources/application.yaml | 1 + 4 files changed, 83 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 32ada255..4881559d 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -84,4 +84,5 @@ private CamelProperties() {} public static final String DUPLICATE_TRANSACTION_LIST = "duplicateTransactionList"; public static final String ORIGINAL_TRANSACTION_LIST = "originalTransactionList"; public static final String CALLBACK = "X-CallbackURL"; + public static final String CONTENT_TYPE = "Content-Type"; } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 8b7e4554..d9a120d1 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -67,6 +67,7 @@ import org.mifos.processor.bulk.config.BudgetAccountConfig; import org.mifos.processor.bulk.config.Program; import org.mifos.processor.bulk.config.RegisteringInstitutionConfig; +import org.mifos.processor.bulk.connectors.service.ProcessorStartRouteService; import org.mifos.processor.bulk.file.FileTransferService; import org.mifos.processor.bulk.properties.TenantImplementation; import org.mifos.processor.bulk.properties.TenantImplementationProperties; @@ -146,8 +147,9 @@ private void setup() { String batchId = UUID.randomUUID().toString(); exchange.setProperty(BATCH_ID, batchId); - }).to("direct:validateFileSyncResponse").choice().when(header("CamelHttpResponseCode").isNotEqualTo("200")) - .log(LoggingLevel.ERROR, "File upload failed").otherwise().to("direct:executeBatch").endChoice().endChoice(); + }).bean(ProcessorStartRouteService.class, "validateFileSyncResponse").choice() + .when(header("CamelHttpResponseCode").isNotEqualTo("200")).log(LoggingLevel.ERROR, "File upload failed").otherwise() + .to("direct:executeBatch").endChoice().endChoice(); from("direct:post-bulk-transfer").unmarshal().mimeMultipart("multipart/*").to("direct:validate-tenant").process(exchange -> { String fileName = System.currentTimeMillis() + "_" + exchange.getIn().getHeader("fileName", String.class); @@ -317,7 +319,7 @@ private void setup() { exchange.getIn().setBody(response.toString()); - }).log("Completed route direct:start-batch-process-csv").to("direct:pollingOutput"); + }).log("Completed route direct:start-batch-process-csv").bean(ProcessorStartRouteService.class, "pollingOutput"); from("direct:start-batch-process-raw").id("direct:start-batch-process-raw").log("Starting route direct:start-batch-process-raw") .process(exchange -> { @@ -328,8 +330,8 @@ private void setup() { exchange.getIn().setBody(response.toString()); }).log("Completed route direct:start-batch-process-raw"); - from("direct:executeBatch").id("direct:executeBatch").log("Starting route direct:executeBatch").to("direct:validate-tenant") - .process(exchange -> { + from("direct:executeBatch").id("direct:executeBatch").log("Starting route direct:executeBatch") + .bean(ProcessorStartRouteService.class, "validateTenant").process(exchange -> { String filename = exchange.getIn().getHeader("filename", String.class); String requestId = exchange.getIn().getHeader("X-CorrelationID", String.class); String purpose = exchange.getIn().getHeader("Purpose", String.class); @@ -455,7 +457,7 @@ private boolean verifyRow(String[] row) { return true; } - private boolean verifyCsv(File csvData) throws IOException { + public boolean verifyCsv(File csvData) throws IOException { BufferedReader br = new BufferedReader(new FileReader(csvData)); String header = br.readLine(); String[] columns = new String[0]; @@ -476,7 +478,7 @@ private boolean verifyCsv(File csvData) throws IOException { return true; } - private void setErrorResponse(Exchange exchange, int responseCode, String errorInfo, String errorDescription) { + public void setErrorResponse(Exchange exchange, int responseCode, String errorInfo, String errorDescription) { // TODO Auto-generated method stub JSONObject json = new JSONObject(); json.put("Error Information: ", errorInfo); @@ -487,7 +489,7 @@ private void setErrorResponse(Exchange exchange, int responseCode, String errorI logger.error("Error response is {}", json); } - private void setResponse(Exchange exchange, int responseCode) { + public void setResponse(Exchange exchange, int responseCode) { exchange.getIn().setHeader(Exchange.HTTP_RESPONSE_CODE, responseCode); } diff --git a/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java new file mode 100644 index 00000000..888ba59f --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java @@ -0,0 +1,71 @@ +package org.mifos.processor.bulk.connectors.service; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.CONTENT_TYPE; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; + +import java.io.File; +import java.io.IOException; +import org.apache.camel.Exchange; +import org.json.JSONObject; +import org.mifos.processor.bulk.camel.routes.ProcessorStartRoute; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +@Service +public class ProcessorStartRouteService { + + @Autowired + ProcessorStartRoute processorStartRoute; + @Value("${csv.size}") + private int csvSize; + @Value("${pollingApi.path}") + private String pollApiPath; + @Value("${pollingApi.timer}") + private String pollApiTimer; + + public Logger logger = LoggerFactory.getLogger(this.getClass()); + + public void validateFileSyncResponse(Exchange exchange) throws IOException { + String fileName = exchange.getIn().getHeader(FILE_NAME, String.class); + File file = new File(fileName); + + // check the file structure + int fileSize = (int) file.length(); + if (fileSize > csvSize) { + processorStartRoute.setErrorResponse(exchange, 400, "File too big", + "The file uploaded is too big. " + "Please upload a file and try again."); + } else if (!processorStartRoute.verifyCsv(file)) { + processorStartRoute.setErrorResponse(exchange, 400, "Invalid file structure", + "The file uploaded contains wrong structure." + " Please upload correct file columns and try again."); + } else { + logger.debug("Filename: {}", fileName); + processorStartRoute.setResponse(exchange, 200); + } + } + + public void validateTenant(Exchange exchange) { + String tenantName = exchange.getIn().getHeader(HEADER_PLATFORM_TENANT_ID, String.class); + // validation is disabled for now + /* + * if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { throw new + * Exception("Invalid tenant value."); } + */ + exchange.setProperty(TENANT_NAME, tenantName); + exchange.getIn().setHeader(CONTENT_TYPE, "application/json;charset=UTF-8"); + } + + public void pollingOutput(Exchange exchange) { + JSONObject json = new JSONObject(); + String pollingPath = String.format("%s%s", pollApiPath, exchange.getProperty(BATCH_ID)); + json.put("PollingPath", pollingPath); + json.put("SuggestedCallbackSeconds", pollApiTimer); + exchange.getIn().setBody(json.toString()); + exchange.getIn().setHeader(Exchange.HTTP_RESPONSE_CODE, 202); + } +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index ead7b933..e3b6a04d 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -121,6 +121,7 @@ callback: url: "http://httpstat.us/503" pollingApi: + path : "/batch/Summary/" timer: "120" callback-phases: From febdbee969dcb756f801d6653fcf1d3f2afbdbc9 Mon Sep 17 00:00:00 2001 From: Sourabh Kumar <148945054+shkr7@users.noreply.github.com> Date: Mon, 4 Mar 2024 13:01:29 +0530 Subject: [PATCH 123/156] [PHEE-587] Convert BD-011 routes to Java/Spring (#151) * [PHEE-587] Convert BD-011 route to Java/Spring * Added spotless change * Added csv and incoming data java routes --- .../camel/routes/ProcessorStartRoute.java | 187 +--------------- .../service/ProcessorStartRouteService.java | 211 ++++++++++++++++++ 2 files changed, 219 insertions(+), 179 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index d9a120d1..3ea44b19 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -1,18 +1,11 @@ package org.mifos.processor.bulk.camel.routes; import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_REQUEST_TYPE; -import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_CLIENT_CORRELATION_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; -import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PROGRAM_ID; -import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.IS_UPDATED; import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; import static org.mifos.processor.bulk.camel.config.CamelProperties.OVERRIDE_HEADER; -import static org.mifos.processor.bulk.camel.config.CamelProperties.PROGRAM_ID; -import static org.mifos.processor.bulk.camel.config.CamelProperties.REGISTERING_INSTITUTE_ID; -import static org.mifos.processor.bulk.camel.config.CamelProperties.RESULT_TRANSACTION_LIST; import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; -import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.APPROVAL_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.AUTHORIZATION_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_AGGREGATE_ENABLED; @@ -20,33 +13,21 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BULK_NOTIF_FAILURE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BULK_NOTIF_SUCCESS; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_URL; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CLIENT_CORRELATION_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_THRESHOLD; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_THRESHOLD_CHECK_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.DE_DUPLICATION_ENABLE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FORMATTING_ENABLED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.IS_FILE_VALID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_CALLBACK_RETRY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_STATUS_RETRY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_ENABLED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.NOTE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERING_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_ENABLED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYER_IDENTIFIER_TYPE; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYER_IDENTIFIER_VALUE; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASES; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASE_COUNT; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PROGRAM_NAME; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_ENABLED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.THRESHOLD_DELAY; -import com.fasterxml.jackson.core.JsonProcessingException; -import io.camunda.zeebe.client.api.command.ClientStatusException; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -54,9 +35,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -65,13 +44,10 @@ import org.apache.tika.Tika; import org.json.JSONObject; import org.mifos.processor.bulk.config.BudgetAccountConfig; -import org.mifos.processor.bulk.config.Program; -import org.mifos.processor.bulk.config.RegisteringInstitutionConfig; import org.mifos.processor.bulk.connectors.service.ProcessorStartRouteService; import org.mifos.processor.bulk.file.FileTransferService; import org.mifos.processor.bulk.properties.TenantImplementation; import org.mifos.processor.bulk.properties.TenantImplementationProperties; -import org.mifos.processor.bulk.schema.Transaction; import org.mifos.processor.bulk.utility.PhaseUtils; import org.mifos.processor.bulk.utility.Utils; import org.mifos.processor.bulk.zeebe.ZeebeProcessStarter; @@ -82,7 +58,6 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; @Component public class ProcessorStartRoute extends BaseRouteBuilder { @@ -181,54 +156,7 @@ private void setup() { // [LOCAL_FILE_PATH] is already set in [direct:validateFileSyncResponse] route .setProperty(LOCAL_FILE_PATH, exchangeProperty(FILE_NAME)).to("direct:get-transaction-array") // make sure new data is set under the exchange variable [RESULT_TRANSACTION_LIST] - .process(exchange -> { - String registeringInstituteId = exchange.getProperty(REGISTERING_INSTITUTE_ID, String.class); - String programId = exchange.getProperty(PROGRAM_ID, String.class); - logger.debug("Inst id: {}, prog id: {}", registeringInstituteId, programId); - if (!(StringUtils.hasText(registeringInstituteId) && StringUtils.hasText(programId))) { - // this will make sure the file is not updated since there is no update in data - logger.debug("InstitutionId or programId is null"); - - exchange.setProperty(IS_UPDATED, false); - return; - } - List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); - logger.debug("Size: {}", transactionList.size()); - RegisteringInstitutionConfig registeringInstitutionConfig = budgetAccountConfig - .getByRegisteringInstituteId(registeringInstituteId); - if (registeringInstitutionConfig == null) { - logger.debug("Element in nested in config: {}", - budgetAccountConfig.getRegisteringInstitutions().get(0).getPrograms().size()); - logger.debug("Registering institute id is null"); - - exchange.setProperty(IS_UPDATED, false); - return; - } - Program program = registeringInstitutionConfig.getByProgramId(programId); - if (program == null) { - // this will make sure the file is not updated since there is no update in data - logger.debug("Program is null"); - exchange.setProperty(IS_UPDATED, false); - return; - } - List resultTransactionList = new ArrayList<>(); - - transactionList.forEach(transaction -> { - transaction.setPayerIdentifierType(program.getIdentifierType()); - transaction.setPayerIdentifier(program.getIdentifierValue()); - resultTransactionList.add(transaction); - try { - logger.debug("Txn: {}", objectMapper.writeValueAsString(transaction)); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }); - exchange.setProperty(RESULT_TRANSACTION_LIST, resultTransactionList); - exchange.setProperty(IS_UPDATED, true); - exchange.setProperty(PROGRAM_NAME, program.getName()); - exchange.setProperty(PAYER_IDENTIFIER_TYPE, program.getIdentifierType()); - exchange.setProperty(PAYER_IDENTIFIER_VALUE, program.getIdentifierValue()); - }).choice() + .bean(ProcessorStartRouteService.class, "updateIncomingData").choice() // update only when previous(edit function) makes any changes to data .when(exchange -> exchange.getProperty(IS_UPDATED, Boolean.class)) // warning: changing this flag can break things @@ -236,90 +164,8 @@ private void setup() { .to("direct:update-file-v2").otherwise().log(LoggingLevel.INFO, "No update"); from("direct:start-batch-process-csv").id("direct:start-batch-process-csv").log("Starting route direct:start-batch-process-csv") - .to("direct:update-incoming-data").process(exchange -> { - String fileName = exchange.getProperty(FILE_NAME, String.class); - String requestId = exchange.getProperty(REQUEST_ID, String.class); - String purpose = exchange.getProperty(PURPOSE, String.class); - String batchId = exchange.getProperty(BATCH_ID, String.class); - String callbackUrl = exchange.getProperty(CALLBACK, String.class); - String note = null; - - if (purpose == null || purpose.isEmpty()) { - purpose = "test payment"; - } - - logger.debug("\n\n Filename: {}", fileName); - logger.debug("\n\n BatchId: {} ", batchId); - - File file = new File(fileName); - file.setWritable(true); - file.setReadable(true); - - logger.debug("File absolute path: {}", file.getAbsolutePath()); - - boolean verifyData = verifyData(file); - logger.debug("Data verification result {}", verifyData); - if (!verifyData) { - note = "Invalid data in file data processing stopped"; - } - - String nm = fileTransferService.uploadFile(file, bucketName); - - logger.debug("File uploaded {}", nm); - - // extracting and setting callback Url - exchange.setProperty(CALLBACK_URL, callbackUrl); - - List phases = phaseUtils.getValues(); - logger.debug(phases.toString()); - Map variables = new HashMap<>(); - variables.put(BATCH_ID, batchId); - variables.put(FILE_NAME, fileName); - variables.put(REQUEST_ID, requestId); - variables.put(PURPOSE, purpose); - variables.put(TENANT_ID, exchange.getProperty(TENANT_NAME)); - variables.put(CALLBACK, callbackUrl); - variables.put(PHASES, phases); - variables.put(PHASE_COUNT, phases.size()); - variables.put(NOTE, note); - variables.put(CLIENT_CORRELATION_ID, exchange.getProperty(CLIENT_CORRELATION_ID)); - variables.put(PROGRAM_NAME, exchange.getProperty(PROGRAM_NAME)); - variables.put(PAYER_IDENTIFIER_TYPE, exchange.getProperty(PAYER_IDENTIFIER_TYPE)); - variables.put(PAYER_IDENTIFIER_VALUE, exchange.getProperty(PAYER_IDENTIFIER_VALUE)); - variables.put(REGISTERING_INSTITUTE_ID, exchange.getProperty(REGISTERING_INSTITUTE_ID)); - variables.put(IS_FILE_VALID, true); - setConfigProperties(variables); - - logger.debug("Zeebe variables published: {}", variables); - log.debug("Variables published to zeebe: {}", variables); - - JSONObject response = new JSONObject(); - String bpmn = getWorkflowForTenant(exchange.getProperty(TENANT_NAME).toString(), "batch-transactions"); - - try { - String tenantSpecificWorkflowId = bpmn.replace("{dfspid}", exchange.getProperty(TENANT_NAME).toString()); - String txnId = zeebeProcessStarter.startZeebeWorkflow(tenantSpecificWorkflowId, "", variables); - if (txnId == null || txnId.isEmpty()) { - response.put("errorCode", 500); - response.put("errorDescription", "Unable to start zeebe workflow"); - response.put("developerMessage", "Issue in starting the zeebe workflow, check the zeebe configuration"); - } else { - response.put("batch_id", batchId); - response.put("request_id", requestId); - response.put("status", "queued"); - } - } catch (ClientStatusException c) { - log.error("Got ClientStatusException : {}", c.getMessage()); - throw c; - } catch (Exception e) { - response.put("errorCode", 500); - response.put("errorDescription", "Unable to start zeebe workflow"); - response.put("developerMessage", e.getLocalizedMessage()); - } - - exchange.getIn().setBody(response.toString()); - - }).log("Completed route direct:start-batch-process-csv").bean(ProcessorStartRouteService.class, "pollingOutput"); + .to("direct:update-incoming-data").bean(ProcessorStartRouteService.class, "startBatchProcessCsv") + .log("Completed route direct:start-batch-process-csv").bean(ProcessorStartRouteService.class, "pollingOutput"); from("direct:start-batch-process-raw").id("direct:start-batch-process-raw").log("Starting route direct:start-batch-process-raw") .process(exchange -> { @@ -331,26 +177,9 @@ private void setup() { }).log("Completed route direct:start-batch-process-raw"); from("direct:executeBatch").id("direct:executeBatch").log("Starting route direct:executeBatch") - .bean(ProcessorStartRouteService.class, "validateTenant").process(exchange -> { - String filename = exchange.getIn().getHeader("filename", String.class); - String requestId = exchange.getIn().getHeader("X-CorrelationID", String.class); - String purpose = exchange.getIn().getHeader("Purpose", String.class); - String type = exchange.getIn().getHeader("Type", String.class); - String clientCorrelationId = exchange.getIn().getHeader(HEADER_CLIENT_CORRELATION_ID, String.class); - String registeringInstitutionId = exchange.getIn().getHeader(HEADER_REGISTERING_INSTITUTE_ID, String.class); - logger.info("registeringInstitutionId {}", registeringInstitutionId); - String programId = exchange.getIn().getHeader(HEADER_PROGRAM_ID, String.class); - String callbackUrl = exchange.getIn().getHeader("X-CallbackURL", String.class); - exchange.setProperty(FILE_NAME, filename); - exchange.setProperty(REQUEST_ID, requestId); - exchange.setProperty(PURPOSE, purpose); - exchange.setProperty(BATCH_REQUEST_TYPE, type); - exchange.setProperty(CLIENT_CORRELATION_ID, clientCorrelationId); - exchange.setProperty(REGISTERING_INSTITUTE_ID, registeringInstitutionId); - exchange.setProperty(PROGRAM_ID, programId); - exchange.setProperty(CALLBACK, callbackUrl); - }).choice().when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("raw")) - .to("direct:start-batch-process-raw") + .bean(ProcessorStartRouteService.class, "validateTenant").bean(ProcessorStartRouteService.class, "executeBatch").choice() + .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("raw")) + .bean(ProcessorStartRouteService.class, "startBatchProcessRaw") .when(exchange -> exchange.getProperty(BATCH_REQUEST_TYPE, String.class).equalsIgnoreCase("csv")) .to("direct:start-batch-process-csv").otherwise() .setBody(exchange -> getUnsupportedTypeJson(exchange.getProperty(BATCH_REQUEST_TYPE, String.class)).toString()) @@ -405,7 +234,7 @@ private void setup() { } - private boolean verifyData(File file) throws IOException { + public boolean verifyData(File file) throws IOException { InputStream ips = new FileInputStream(file); InputStreamReader ipsr = new InputStreamReader(ips); BufferedReader br = new BufferedReader(ipsr); @@ -493,7 +322,7 @@ public void setResponse(Exchange exchange, int responseCode) { exchange.getIn().setHeader(Exchange.HTTP_RESPONSE_CODE, responseCode); } - private Map setConfigProperties(Map variables) { + public Map setConfigProperties(Map variables) { variables.put(BATCH_AGGREGATE_ENABLED, workerConfig.isBatchAggregateEnabled); variables.put(PARTY_LOOKUP_ENABLED, workerConfig.isPartyLookUpWorkerEnabled); variables.put(AUTHORIZATION_ENABLED, workerConfig.isAuthorizationWorkerEnabled); diff --git a/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java index 888ba59f..b5125159 100644 --- a/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java +++ b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java @@ -1,27 +1,79 @@ package org.mifos.processor.bulk.connectors.service; +import static org.mifos.processor.bulk.camel.config.CamelProperties.BATCH_REQUEST_TYPE; import static org.mifos.processor.bulk.camel.config.CamelProperties.CONTENT_TYPE; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_CLIENT_CORRELATION_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PROGRAM_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.IS_UPDATED; +import static org.mifos.processor.bulk.camel.config.CamelProperties.PROGRAM_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.REGISTERING_INSTITUTE_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.RESULT_TRANSACTION_LIST; import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_URL; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CLIENT_CORRELATION_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.IS_FILE_VALID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.NOTE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYER_IDENTIFIER_TYPE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYER_IDENTIFIER_VALUE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASES; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASE_COUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PROGRAM_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.camunda.zeebe.client.api.command.ClientStatusException; import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; import org.apache.camel.Exchange; import org.json.JSONObject; import org.mifos.processor.bulk.camel.routes.ProcessorStartRoute; +import org.mifos.processor.bulk.config.BudgetAccountConfig; +import org.mifos.processor.bulk.config.Program; +import org.mifos.processor.bulk.config.RegisteringInstitutionConfig; +import org.mifos.processor.bulk.file.FileTransferService; +import org.mifos.processor.bulk.schema.Transaction; +import org.mifos.processor.bulk.utility.PhaseUtils; +import org.mifos.processor.bulk.zeebe.ZeebeProcessStarter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; @Service public class ProcessorStartRouteService { @Autowired ProcessorStartRoute processorStartRoute; + @Autowired + @Qualifier("awsStorage") + private FileTransferService fileTransferService; + @Autowired + private ZeebeProcessStarter zeebeProcessStarter; + @Autowired + PhaseUtils phaseUtils; + @Autowired + public ObjectMapper objectMapper; + @Autowired + BudgetAccountConfig budgetAccountConfig; + @Value("${application.bucket-name}") + private String bucketName; @Value("${csv.size}") private int csvSize; @Value("${pollingApi.path}") @@ -68,4 +120,163 @@ public void pollingOutput(Exchange exchange) { exchange.getIn().setBody(json.toString()); exchange.getIn().setHeader(Exchange.HTTP_RESPONSE_CODE, 202); } + + public void executeBatch(Exchange exchange) { + String filename = exchange.getIn().getHeader("filename", String.class); + String requestId = exchange.getIn().getHeader("X-CorrelationID", String.class); + String purpose = exchange.getIn().getHeader("Purpose", String.class); + String type = exchange.getIn().getHeader("Type", String.class); + String clientCorrelationId = exchange.getIn().getHeader(HEADER_CLIENT_CORRELATION_ID, String.class); + String registeringInstitutionId = exchange.getIn().getHeader(HEADER_REGISTERING_INSTITUTE_ID, String.class); + logger.info("registeringInstitutionId {}", registeringInstitutionId); + String programId = exchange.getIn().getHeader(HEADER_PROGRAM_ID, String.class); + String callbackUrl = exchange.getIn().getHeader("X-CallbackURL", String.class); + exchange.setProperty(FILE_NAME, filename); + exchange.setProperty(REQUEST_ID, requestId); + exchange.setProperty(PURPOSE, purpose); + exchange.setProperty(BATCH_REQUEST_TYPE, type); + exchange.setProperty(CLIENT_CORRELATION_ID, clientCorrelationId); + exchange.setProperty(REGISTERING_INSTITUTE_ID, registeringInstitutionId); + exchange.setProperty(PROGRAM_ID, programId); + exchange.setProperty(CALLBACK, callbackUrl); + } + + public void startBatchProcessRaw(Exchange exchange) { + JSONObject response = new JSONObject(); + response.put("batch_id", UUID.randomUUID().toString()); + response.put("request_id", UUID.randomUUID().toString()); + response.put("status", "queued"); + exchange.getIn().setBody(response.toString()); + } + + public void updateIncomingData(Exchange exchange) { + String registeringInstituteId = exchange.getProperty(REGISTERING_INSTITUTE_ID, String.class); + String programId = exchange.getProperty(PROGRAM_ID, String.class); + logger.debug("Inst id: {}, prog id: {}", registeringInstituteId, programId); + if (!(StringUtils.hasText(registeringInstituteId) && StringUtils.hasText(programId))) { + // this will make sure the file is not updated since there is no update in data + logger.debug("InstitutionId or programId is null"); + + exchange.setProperty(IS_UPDATED, false); + return; + } + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + logger.debug("Size: {}", transactionList.size()); + RegisteringInstitutionConfig registeringInstitutionConfig = budgetAccountConfig.getByRegisteringInstituteId(registeringInstituteId); + if (registeringInstitutionConfig == null) { + logger.debug("Element in nested in config: {}", budgetAccountConfig.getRegisteringInstitutions().get(0).getPrograms().size()); + logger.debug("Registering institute id is null"); + + exchange.setProperty(IS_UPDATED, false); + return; + } + Program program = registeringInstitutionConfig.getByProgramId(programId); + if (program == null) { + // this will make sure the file is not updated since there is no update in data + logger.debug("Program is null"); + exchange.setProperty(IS_UPDATED, false); + return; + } + List resultTransactionList = new ArrayList<>(); + + transactionList.forEach(transaction -> { + transaction.setPayerIdentifierType(program.getIdentifierType()); + transaction.setPayerIdentifier(program.getIdentifierValue()); + resultTransactionList.add(transaction); + try { + logger.debug("Txn: {}", objectMapper.writeValueAsString(transaction)); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }); + exchange.setProperty(RESULT_TRANSACTION_LIST, resultTransactionList); + exchange.setProperty(IS_UPDATED, true); + exchange.setProperty(PROGRAM_NAME, program.getName()); + exchange.setProperty(PAYER_IDENTIFIER_TYPE, program.getIdentifierType()); + exchange.setProperty(PAYER_IDENTIFIER_VALUE, program.getIdentifierValue()); + } + + public void startBatchProcessCsv(Exchange exchange) throws IOException { + String fileName = exchange.getProperty(FILE_NAME, String.class); + String requestId = exchange.getProperty(REQUEST_ID, String.class); + String purpose = exchange.getProperty(PURPOSE, String.class); + String batchId = exchange.getProperty(BATCH_ID, String.class); + String callbackUrl = exchange.getProperty(CALLBACK, String.class); + String note = null; + + if (purpose == null || purpose.isEmpty()) { + purpose = "test payment"; + } + + logger.debug("\n\n Filename: {}", fileName); + logger.debug("\n\n BatchId: {} ", batchId); + + File file = new File(fileName); + file.setWritable(true); + file.setReadable(true); + + logger.debug("File absolute path: {}", file.getAbsolutePath()); + + boolean verifyData = processorStartRoute.verifyData(file); + logger.debug("Data verification result {}", verifyData); + if (!verifyData) { + note = "Invalid data in file data processing stopped"; + } + + String nm = fileTransferService.uploadFile(file, bucketName); + + logger.debug("File uploaded {}", nm); + + // extracting and setting callback Url + exchange.setProperty(CALLBACK_URL, callbackUrl); + + List phases = phaseUtils.getValues(); + logger.debug(phases.toString()); + Map variables = new HashMap<>(); + variables.put(BATCH_ID, batchId); + variables.put(FILE_NAME, fileName); + variables.put(REQUEST_ID, requestId); + variables.put(PURPOSE, purpose); + variables.put(TENANT_ID, exchange.getProperty(TENANT_NAME)); + variables.put(CALLBACK, callbackUrl); + variables.put(PHASES, phases); + variables.put(PHASE_COUNT, phases.size()); + variables.put(NOTE, note); + variables.put(CLIENT_CORRELATION_ID, exchange.getProperty(CLIENT_CORRELATION_ID)); + variables.put(PROGRAM_NAME, exchange.getProperty(PROGRAM_NAME)); + variables.put(PAYER_IDENTIFIER_TYPE, exchange.getProperty(PAYER_IDENTIFIER_TYPE)); + variables.put(PAYER_IDENTIFIER_VALUE, exchange.getProperty(PAYER_IDENTIFIER_VALUE)); + variables.put(REGISTERING_INSTITUTE_ID, exchange.getProperty(REGISTERING_INSTITUTE_ID)); + variables.put(IS_FILE_VALID, true); + processorStartRoute.setConfigProperties(variables); + + logger.debug("Zeebe variables published: {}", variables); + logger.debug("Variables published to zeebe: {}", variables); + + JSONObject response = new JSONObject(); + String bpmn = processorStartRoute.getWorkflowForTenant(exchange.getProperty(TENANT_NAME).toString(), "batch-transactions"); + + try { + String tenantSpecificWorkflowId = bpmn.replace("{dfspid}", exchange.getProperty(TENANT_NAME).toString()); + String txnId = zeebeProcessStarter.startZeebeWorkflow(tenantSpecificWorkflowId, "", variables); + if (txnId == null || txnId.isEmpty()) { + response.put("errorCode", 500); + response.put("errorDescription", "Unable to start zeebe workflow"); + response.put("developerMessage", "Issue in starting the zeebe workflow, check the zeebe configuration"); + } else { + response.put("batch_id", batchId); + response.put("request_id", requestId); + response.put("status", "queued"); + } + } catch (ClientStatusException c) { + logger.error("Got ClientStatusException : {}", c.getMessage()); + throw c; + } catch (Exception e) { + response.put("errorCode", 500); + response.put("errorDescription", "Unable to start zeebe workflow"); + response.put("developerMessage", e.getLocalizedMessage()); + } + + exchange.getIn().setBody(response.toString()); + } } From 4c688a1462d7dfb2d48f3fa8e648778b23ef2126 Mon Sep 17 00:00:00 2001 From: Manoj <56669674+fynmanoj@users.noreply.github.com> Date: Wed, 6 Mar 2024 19:09:19 +0530 Subject: [PATCH 124/156] update-debulking-dfspid --- src/main/resources/application.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index e3b6a04d..b2c65cb6 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -195,7 +195,7 @@ payment-mode: - id: "CLOSEDLOOP" type: "BULK" endpoint: "bulk_connector_{MODE}-{dfspid}" - debulkingDfspid: "lion" + debulkingDfspid: "wakanda" batch-authorization: callback-url: "https://ph-ee-connector-bulk:8443/authorization/callback" From f8908c6f3ccec73a77d177203f03409b16859620 Mon Sep 17 00:00:00 2001 From: Dhaval Maniyar <100835289+logoutdhaval@users.noreply.github.com> Date: Fri, 8 Mar 2024 19:32:22 +0530 Subject: [PATCH 125/156] [PHEE-615] Create Add batchId and correlationId in batch callback (#155) * [PHEE-615] Create Add batchId and correlationId in batch callback * spotless fix --- .../bulk/camel/routes/SendCallbackRoute.java | 13 +++++++++---- .../processor/bulk/schema/BatchCallbackDTO.java | 17 +++++++++++++++++ .../bulk/zeebe/worker/AggregateWorker.java | 2 ++ 3 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/schema/BatchCallbackDTO.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java index 2244e448..177a0a9d 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java @@ -1,9 +1,11 @@ package org.mifos.processor.bulk.camel.routes; import static org.mifos.processor.bulk.camel.config.CamelProperties.CALLBACK_RESPONSE_CODE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_RETRY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_SUCCESS; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CLIENT_CORRELATION_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_RATE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_CODE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_DESCRIPTION; @@ -14,6 +16,7 @@ import java.util.List; import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; +import org.mifos.processor.bulk.schema.BatchCallbackDTO; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -35,14 +38,16 @@ public void configure() throws Exception { from(RouteId.SEND_CALLBACK.getValue()).id(RouteId.SEND_CALLBACK.getValue()).log("Starting route " + RouteId.SEND_CALLBACK.name()) .log("Sending callback for Batch Processing").setHeader(Exchange.HTTP_METHOD, constant("POST")).process(exchange -> { - String body = String.format("The Batch Aggregation API was complete with : %s", + String message = String.format("The Batch Aggregation API was complete with : %s", exchange.getProperty(COMPLETION_RATE).toString()); callbackUrl = exchange.getProperty(CALLBACK, String.class); logger.info("Callback URL: {}", callbackUrl); - logger.info("Callback Body: {}", body); - + logger.info("Callback Body: {}", message); + String batchId = exchange.getProperty(BATCH_ID, String.class); + String clientCorrelationId = exchange.getProperty(CLIENT_CORRELATION_ID, String.class); + BatchCallbackDTO batchCallbackDTO = new BatchCallbackDTO(clientCorrelationId, batchId, message); ObjectMapper objectMapper = new ObjectMapper(); - String jsonString = objectMapper.writeValueAsString(body); + String jsonString = objectMapper.writeValueAsString(batchCallbackDTO); exchange.getIn().setBody(jsonString); }).choice().when(exchangeProperty("X-CallbackURL").isNotNull()).setHeader(Exchange.HTTP_METHOD, constant("POST")) .toD("${exchangeProperty.X-CallbackURL}?bridgeEndpoint=true&throwExceptionOnFailure=false") diff --git a/src/main/java/org/mifos/processor/bulk/schema/BatchCallbackDTO.java b/src/main/java/org/mifos/processor/bulk/schema/BatchCallbackDTO.java new file mode 100644 index 00000000..735890dd --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/schema/BatchCallbackDTO.java @@ -0,0 +1,17 @@ +package org.mifos.processor.bulk.schema; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class BatchCallbackDTO { + + String clientCorrelationId; + String batchId; + String message; +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java index 66272d44..9bdd5d3f 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/AggregateWorker.java @@ -4,6 +4,7 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_RETRY; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CLIENT_CORRELATION_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_RATE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_CODE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_DESCRIPTION; @@ -34,6 +35,7 @@ public void setup() { int successRate = (int) variables.getOrDefault(COMPLETION_RATE, 0); Exchange exchange = new DefaultExchange(camelContext); + exchange.setProperty(CLIENT_CORRELATION_ID, variables.get(CLIENT_CORRELATION_ID)); exchange.setProperty(BATCH_ID, variables.get(BATCH_ID)); exchange.setProperty(TENANT_ID, variables.get(TENANT_ID)); exchange.setProperty(MAX_CALLBACK_RETRY, variables.get(MAX_CALLBACK_RETRY)); From 73239ef9d80139710313f1fe722c52d1270ac294 Mon Sep 17 00:00:00 2001 From: Dhaval Maniyar <100835289+logoutdhaval@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:24:32 +0530 Subject: [PATCH 126/156] [PHEE-617] Getting multiple callbacks on single batch transaction API call (#156) --- .../bulk/camel/routes/BatchAggregateRoute.java | 2 -- .../bulk/zeebe/worker/SendCallbackWorker.java | 14 +++++++++++++- src/main/resources/application.yaml | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java index 17b92c09..403913d2 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java @@ -65,8 +65,6 @@ public void configure() throws Exception { } exchange.setProperty(COMPLETION_RATE, percentage); - - producerTemplate.send(RouteId.SEND_CALLBACK.getValue(), exchange); }).otherwise().log(LoggingLevel.ERROR, "Batch aggregate request unsuccessful").process(exchange -> { exchange.setProperty(BATCH_STATUS_FAILED, true); exchange.setProperty(ERROR_DESCRIPTION, exchange.getIn().getBody(String.class)); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java index a34df04b..723e64b2 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SendCallbackWorker.java @@ -1,16 +1,20 @@ package org.mifos.processor.bulk.zeebe.worker; import static org.mifos.processor.bulk.camel.config.CamelProperties.CALLBACK_RESPONSE_CODE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_RETRY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CALLBACK_SUCCESS; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.CLIENT_CORRELATION_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_RATE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.COMPLETION_THRESHOLD; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_CODE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ERROR_DESCRIPTION; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_CALLBACK_RETRY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MAX_STATUS_RETRY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASES; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASE_COUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.RETRY; import java.util.Map; import org.apache.camel.Exchange; @@ -41,7 +45,15 @@ public void setup() { exchange.setProperty(COMPLETION_RATE, variables.get(COMPLETION_RATE)); exchange.setProperty(PHASES, variables.get(PHASES)); exchange.setProperty(PHASE_COUNT, variables.get(PHASE_COUNT)); - sendToCamelRoute(RouteId.SEND_CALLBACK, exchange); + exchange.setProperty(BATCH_ID, variables.get(BATCH_ID)); + exchange.setProperty(CLIENT_CORRELATION_ID, variables.get(CLIENT_CORRELATION_ID)); + Integer maxRetry = Integer.parseInt(variables.get(MAX_STATUS_RETRY).toString()); + Integer completionRate = Integer.parseInt(variables.get(COMPLETION_RATE).toString()); + Integer completionThreshold = Integer.parseInt(variables.get(COMPLETION_THRESHOLD).toString()); + Integer statusRetry = Integer.parseInt(variables.get(RETRY).toString()); + if (statusRetry >= maxRetry || completionRate >= completionThreshold) { + sendToCamelRoute(RouteId.SEND_CALLBACK, exchange); + } } Boolean callbackSuccess = exchange.getProperty(CALLBACK_SUCCESS, Boolean.class); if (callbackSuccess == null || !callbackSuccess) { diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index b2c65cb6..c4012d69 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -111,7 +111,7 @@ config: completion-threshold-check: enable: false completion-threshold: 95 # in percentage - max-retry: 15 #can be as high as 30 + max-retry: 3 #can be as high as 30 delay: 2 # in seconds deduplication: enabled: true From 02725b385da8846391600de1e772908806aff0c1 Mon Sep 17 00:00:00 2001 From: Anover000 <140409886+Anover000@users.noreply.github.com> Date: Sun, 17 Mar 2024 17:05:20 +0530 Subject: [PATCH 127/156] modification in batch raw dto: subType->paymentMode, add request id (#159) --- .../processor/bulk/format/RestRequestConvertor.java | 11 +++++++---- .../mifos/processor/bulk/schema/BatchRequestDTO.java | 3 ++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/format/RestRequestConvertor.java b/src/main/java/org/mifos/processor/bulk/format/RestRequestConvertor.java index 684635ef..8186bafe 100644 --- a/src/main/java/org/mifos/processor/bulk/format/RestRequestConvertor.java +++ b/src/main/java/org/mifos/processor/bulk/format/RestRequestConvertor.java @@ -29,7 +29,8 @@ public BatchRequestDTO convertTo(Transaction object) { batchRequestDTO.setCreditParty(List.of(creditParty)); batchRequestDTO.setDebitParty(List.of(debitParty)); batchRequestDTO.setDescriptionText(object.getNote()); - batchRequestDTO.setSubType(object.getPaymentMode()); + batchRequestDTO.setPaymentMode(object.getPaymentMode()); + batchRequestDTO.setRequestId(object.getRequestId()); return batchRequestDTO; } @@ -50,8 +51,8 @@ public Transaction convertFrom(BatchRequestDTO object) { transaction.setPayeeIdentifier(object.getCreditParty().get(0).getValue()); } transaction.setNote(object.getDescriptionText()); - transaction.setPaymentMode(object.getSubType()); - + transaction.setPaymentMode(object.getPaymentMode()); + transaction.setRequestId(object.getRequestId()); return transaction; } @@ -68,7 +69,9 @@ public List convertListFrom(List objects) { for (int i = 0; i < objects.size(); i++) { Transaction transaction = convertFrom(objects.get(i)); transaction.setId(i); - transaction.setRequestId(UUID.randomUUID().toString()); + if (transaction.getRequestId() == null) { + transaction.setRequestId(UUID.randomUUID().toString()); + } transactionList.add(transaction); } return transactionList; diff --git a/src/main/java/org/mifos/processor/bulk/schema/BatchRequestDTO.java b/src/main/java/org/mifos/processor/bulk/schema/BatchRequestDTO.java index 85d1551d..3ccce3f5 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/BatchRequestDTO.java +++ b/src/main/java/org/mifos/processor/bulk/schema/BatchRequestDTO.java @@ -12,10 +12,11 @@ @NoArgsConstructor public class BatchRequestDTO { + String requestId; List creditParty; List debitParty; - String subType; + String paymentMode; String amount; String currency; String descriptionText; From b98aa058e2816f6ec5b9f6e916a80d80382dfc96 Mon Sep 17 00:00:00 2001 From: somanath21 <104554892+somanath21@users.noreply.github.com> Date: Mon, 1 Apr 2024 19:59:37 +0530 Subject: [PATCH 128/156] [GOV-134] Enable management endpoints for health checks (#162) * enable liveness and readiness * expose management endpoints for health checks * rebase * rebase --------- Co-authored-by: Somanath Hugar --- src/main/resources/application.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index c4012d69..9e67e12f 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -171,9 +171,11 @@ budget-account: identifierType: "ACCOUNT" identifierValue: "123456789" + management: endpoint: - health: + health: + enabled: true probes: enabled: true liveness: From 829d419890ce1b1910bb4684238cdfed8ff3ffdb Mon Sep 17 00:00:00 2001 From: abhinavm117 <150892830+abhinavm117@users.noreply.github.com> Date: Mon, 8 Apr 2024 10:45:24 +0530 Subject: [PATCH 129/156] [PHEE-641] Inconsistent behaviour Fix (#163) * Changing account lookup bpmn to lion * Fix splitting * checkstyle fix * BatchId in transfers table logic updtae Removed unnecssary cvs enabled kafka port * Added dockerfile, and rmeoved commented code Spliiting on Spliiting on * Fixed spotless issues * BatchId in transfers table logic updtae Removed unnecssary cvs enabled kafka port * Fixed spotless issues * yaml file configiration changes * Updated service url * updatetd csv schema * deleted csv fies * checkstyle changes * Init sub-batch fix * raw body fix * Delete c20db108-f2fe-4151-a526-b353cee18e41_bulk_payment.csv Unnecessary file, review comment * test * debug * Resolved Review comments --------- Co-authored-by: Apurb Rajdhan <103931815+apurbraj@users.noreply.github.com> Co-authored-by: abhinav Co-authored-by: Abhinav Mishra Co-authored-by: Anover000 --- Dockerfile | 2 +- .../bulk/camel/routes/InitSubBatchRoute.java | 1 - .../bulk/camel/routes/SplittingRoute.java | 66 ++++++++++++++----- .../processor/bulk/schema/Transaction.java | 27 +++++++- .../bulk/utility/TransactionParser.java | 66 +++++++++++++++++++ .../BatchAccountLookupCallbackWorker.java | 1 + .../bulk/zeebe/worker/InitSubBatchWorker.java | 15 +++-- .../bulk/zeebe/worker/SplittingWorker.java | 3 + src/main/resources/application.yaml | 6 +- 9 files changed, 155 insertions(+), 32 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/utility/TransactionParser.java diff --git a/Dockerfile b/Dockerfile index 4bad37e3..ac38c28a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,4 +2,4 @@ FROM openjdk:17 EXPOSE 5000 COPY build/libs/*.jar . -CMD java -jar *.jar +CMD java -jar *.jar \ No newline at end of file diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index 8983c5bf..5b876233 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -115,7 +115,6 @@ public void configure() throws Exception { PaymentModeMapping mapping = paymentModeConfiguration.getByMode(paymentMode); String tenantName = exchange.getProperty(TENANT_NAME, String.class); - tenantName = mapping.getDebulkingDfspid() == null ? tenantName : mapping.getDebulkingDfspid(); Map variables = exchange.getProperty(ZEEBE_VARIABLE, Map.class); variables.put(PAYMENT_MODE, paymentMode); variables.put(DEBULKINGDFSPID, mapping.getDebulkingDfspid() == null ? tenantName : mapping.getDebulkingDfspid()); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java index 8edfd624..269d59b7 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java @@ -1,7 +1,9 @@ package org.mifos.processor.bulk.camel.routes; import static org.mifos.processor.bulk.camel.config.CamelProperties.LOCAL_FILE_PATH; +import static org.mifos.processor.bulk.camel.config.CamelProperties.OVERRIDE_HEADER; import static org.mifos.processor.bulk.camel.config.CamelProperties.REGISTERING_INSTITUTE_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.RESULT_TRANSACTION_LIST; import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_SUB_BATCH_FILE_NAME_ARRAY; import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_COUNT; @@ -22,7 +24,7 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileReader; -import java.io.FileWriter; +import java.io.IOException; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -34,6 +36,7 @@ import org.apache.camel.LoggingLevel; import org.mifos.processor.bulk.schema.SubBatchEntity; import org.mifos.processor.bulk.schema.Transaction; +import org.mifos.processor.bulk.utility.TransactionParser; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -46,7 +49,7 @@ public class SplittingRoute extends BaseRouteBuilder { @Autowired private CsvMapper csvMapper; @Value("${config.partylookup.enable}") - private boolean partyLookupEnabled; + private boolean isPartyLookupEnabled; @Override public void configure() throws Exception { @@ -69,31 +72,44 @@ public void configure() throws Exception { List subBatchFile = new ArrayList<>(); Set distinctPayeeIds = transactionList.stream().map(Transaction::getPayeeDfspId).collect(Collectors.toSet()); logger.info("Payee id {}", distinctPayeeIds); - if (partyLookupEnabled && !distinctPayeeIds.isEmpty()) { + logger.info("Number of payeeId {}", distinctPayeeIds.size()); + Boolean isBatchAccountLookupEnabled = (Boolean) exchange.getProperty("batchAccountLookup"); + if (isPartyLookupEnabled && isBatchAccountLookupEnabled) { // Create a map to store transactions for each payeeid Map> transactionsByPayeeId = new HashMap<>(); // Split the list based on distinct payeeids + Map> subBatchIdMap = new HashMap<>(); + List subBatchIdList = new ArrayList<>(); + Map transactionBatchMap = new HashMap<>(); for (String payeeId : distinctPayeeIds) { List transactionsForPayee = transactionList.stream() .filter(transaction -> payeeId.equals(transaction.getPayeeDfspId())).collect(Collectors.toList()); + String subBatchId = UUID.randomUUID().toString(); transactionsByPayeeId.put(payeeId, transactionsForPayee); - } - for (String payeeId : distinctPayeeIds) { - List transactionsForSpecificPayee = transactionsByPayeeId.get(payeeId); - String filename = UUID.randomUUID() + "_" + "sub-batch-" + payeeId + ".csv"; + // Assign batch IDs to transactions and store in a map for easy lookup + transactionsForPayee.forEach(transaction -> { + transaction.setBatchId(subBatchId); + transactionBatchMap.put(transaction, subBatchId); + }); + + // Create CSV file for the current payee + String filename = UUID.randomUUID() + "_sub-batch-" + payeeId + ".csv"; logger.info("Created sub-batch with file name {}", filename); - CsvSchema csvSchema = csvMapper.schemaFor(Transaction.class); - csvSchema = csvSchema.withHeader(); + CsvSchema csvSchema = csvMapper.schemaFor(Transaction.class).withHeader(); File file = new File(filename); SequenceWriter writer = csvMapper.writerWithSchemaFor(Transaction.class).with(csvSchema).writeValues(file); - for (Transaction transaction : transactionsForSpecificPayee) { + for (Transaction transaction : transactionsForPayee) { writer.write(transaction); } subBatchFile.add(filename); } + // Set properties + transactionList.forEach(transaction -> transaction.setBatchId(transactionBatchMap.get(transaction))); + exchange.setProperty(RESULT_TRANSACTION_LIST, transactionList); + exchange.setProperty(TRANSACTION_LIST, transactionList); } else { List lines = new ArrayList<>(); String line = null; @@ -110,16 +126,30 @@ public void configure() throws Exception { } int subBatchCount = 1; + CsvSchema csvSchema = csvMapper.schemaFor(Transaction.class); + csvSchema = csvSchema.withHeader(); for (int i = 0; i < lines.size(); i += subBatchSize) { + String subBatchId = UUID.randomUUID().toString(); String filename = UUID.randomUUID() + "_" + "sub-batch-" + subBatchCount + ".csv"; - FileWriter writer = new FileWriter(filename); - writer.write(header); + logger.info("SubBatch Id {}", subBatchId); + + List subBatchTransactions = new ArrayList<>(); for (int j = i; j < Math.min(i + subBatchSize, lines.size()); j++) { - writer.write(lines.get(j) + System.lineSeparator()); + Transaction transaction = TransactionParser.parseLineToTransaction(lines.get(j)); + assert transaction != null; + transaction.setBatchId(subBatchId); // Set the subBatchId for the transaction + subBatchTransactions.add(transaction); + } + + // Write the list of Transactions to the file + File file = new File(filename); + try (SequenceWriter writer = csvMapper.writer(csvSchema).writeValues(file)) { + writer.writeAll(subBatchTransactions); + } catch (IOException e) { + logger.error("Failed to write sub-batch file: " + filename, e); } - writer.close(); logger.info("Created sub-batch with file name {}", filename); - subBatchFile.add(filename); + subBatchFile.add(filename); // Ensure this list is declared and accessible subBatchCount++; } } @@ -127,7 +157,9 @@ public void configure() throws Exception { exchange.setProperty(SUB_BATCH_COUNT, subBatchFile.size()); exchange.setProperty(SUB_BATCH_CREATED, true); exchange.setProperty(SERVER_SUB_BATCH_FILE_NAME_ARRAY, new ArrayList()); - }); + }).log("updating orignal").setProperty(LOCAL_FILE_PATH, exchangeProperty(SERVER_FILE_NAME)) + .setProperty(OVERRIDE_HEADER, constant(true)) // default header in CSV file will be used + .to("direct:update-file-v2").to("direct:upload-file"); // Iterate through each CSVs of sub-batches and uploads in cloud from("direct:upload-sub-batch-file").id("direct:upload-sub-batch-file").log("Starting upload of sub-batch file") @@ -165,7 +197,7 @@ public void configure() throws Exception { SubBatchEntity subBatchEntity = getDefaultSubBatchEntity(); subBatchEntity.setBatchId((String) zeebeVariables.get(BATCH_ID)); - subBatchEntity.setSubBatchId(UUID.randomUUID().toString()); + subBatchEntity.setSubBatchId(transactionList.get(0).getBatchId()); subBatchEntity.setRequestId((String) zeebeVariables.get(REQUEST_ID)); subBatchEntity.setCorrelationId((String) zeebeVariables.get(CLIENT_CORRELATION_ID)); subBatchEntity.setPayerFsp((String) zeebeVariables.get(PAYER_IDENTIFIER)); diff --git a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java index 7c8f9514..85e70800 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java +++ b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java @@ -4,14 +4,15 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import java.util.Objects; import lombok.Getter; import lombok.Setter; @Getter @Setter @JsonIgnoreProperties(ignoreUnknown = true) -@JsonPropertyOrder({ "id", "request_id", "payment_mode", "account_number", "payer_identifier_type", "payer_identifier", - "payee_identifier_type", "payee_identifier", "amount", "currency", "note", "program_shortcode", "cycle", "payee_dfsp_id" }) +@JsonPropertyOrder({ "id", "request_id", "payment_mode", "payer_identifier_type", "payer_identifier", "payee_identifier_type", + "payee_identifier", "amount", "currency", "note", "program_shortcode", "cycle", "payee_dfsp_id", "batch_id", "account_number" }) public class Transaction implements CsvSchema { @JsonProperty("id") @@ -32,6 +33,26 @@ public class Transaction implements CsvSchema { @JsonProperty("currency") private String currency; + @Override + public boolean equals(Object transaction) { + if (this == transaction) { + return true; + } + if ((transaction == null) || (getClass() != transaction.getClass())) { + return false; + } + Transaction that = (Transaction) transaction; + return (id == that.id) && (Objects.equals(requestId, that.requestId)) && (Objects.equals(paymentMode, that.paymentMode)) + && (Objects.equals(accountNumber, that.accountNumber)) && (Objects.equals(amount, that.amount)) + && (Objects.equals(payeeDfspId, that.payeeDfspId)); + } + + @Override + public int hashCode() { + return Objects.hash(id, requestId, paymentMode, accountNumber, amount, currency, note, payerIdentifierType, payerIdentifier, + payeeIdentifierType, payeeIdentifier, payeeDfspId); + } + @JsonProperty("note") private String note; @@ -56,7 +77,7 @@ public class Transaction implements CsvSchema { @JsonProperty("payee_dfsp_id") private String payeeDfspId; - @JsonIgnore + @JsonProperty("batch_id") private String batchId; @Override diff --git a/src/main/java/org/mifos/processor/bulk/utility/TransactionParser.java b/src/main/java/org/mifos/processor/bulk/utility/TransactionParser.java new file mode 100644 index 00000000..03f4138f --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/utility/TransactionParser.java @@ -0,0 +1,66 @@ +package org.mifos.processor.bulk.utility; + +import org.mifos.processor.bulk.schema.Transaction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class TransactionParser { + + private static final Logger logger = LoggerFactory.getLogger(TransactionParser.class); + + private TransactionParser() { + throw new IllegalStateException("Utility class"); + } + + public static Transaction parseLineToTransaction(String line) { + try { + String[] parts = line.split(",", -1); + Transaction transaction = new Transaction(); + + if (parts.length > 0 && !parts[0].isEmpty()) { + transaction.setId(Integer.parseInt(parts[0])); + } + if (parts.length > 1) { + transaction.setRequestId(parts[1]); + } + if (parts.length > 2) { + transaction.setPaymentMode(parts[2]); + } + if (parts.length > 4) { + transaction.setPayerIdentifierType(parts[3]); + } + if (parts.length > 5) { + transaction.setPayerIdentifier(parts[4]); + } + if (parts.length > 6) { + transaction.setPayeeIdentifierType(parts[5]); + } + if (parts.length > 7) { + transaction.setPayeeIdentifier(parts[6]); + } + if (parts.length > 8) { + transaction.setAmount(parts[7]); + } + if (parts.length > 9) { + transaction.setCurrency(parts[8]); + } + if (parts.length > 10) { + transaction.setNote(parts[9]); + } + if (parts.length > 11) { + transaction.setProgramShortCode(parts[10]); + } + if (parts.length > 12) { + transaction.setCycle(parts[11]); + } + if (parts.length > 13) { + transaction.setPayeeDfspId(parts[12]); + } + + return transaction; + } catch (Exception e) { + logger.error("Error parsing line to Transaction object: {}", line, e); + return null; + } + } +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupCallbackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupCallbackWorker.java index 4a94084e..f99c070b 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupCallbackWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupCallbackWorker.java @@ -34,6 +34,7 @@ public void setup() { String filename = (String) variables.get(FILE_NAME); String batchAccountLookupCallback = (String) variables.get("batchAccountLookupCallback"); variables.put(PARTY_LOOKUP_FAILED, false); + variables.put("batchAccountLookup", true); exchange.setProperty(SERVER_FILE_NAME, filename); exchange.setProperty("batchAccountLookupCallback", batchAccountLookupCallback); exchange.setProperty("workflowInstanceKey", job.getProcessInstanceKey()); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java index 2acacf11..2f31567d 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java @@ -72,15 +72,16 @@ public void setup() { String fileName = subBatches.remove(0); SubBatchEntity subBatchEntity = null; - - for (SubBatchEntity subBatch : subBatchEntityList) { - if (subBatch.getRequestFile().contains(fileName)) { - subBatchEntity = subBatch; - logger.info("SubBatchEntity found"); + if (isSplittingEnabled) { + for (SubBatchEntity subBatch : subBatchEntityList) { + if (subBatch.getRequestFile().contains(fileName)) { + subBatchEntity = subBatch; + logger.info("SubBatchEntity found"); + break; + } } + logger.debug("BatchEntity for this subbatch is {}", objectMapper.writeValueAsString(subBatchEntity)); } - logger.debug("BatchEntity for this subbatch is {}", objectMapper.writeValueAsString(subBatchEntity)); - Exchange exchange = new DefaultExchange(camelContext); exchange.setProperty(TENANT_NAME, variables.get(TENANT_ID)); exchange.setProperty(SERVER_FILE_NAME, fileName); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java index 8ba21146..f3bd3088 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/SplittingWorker.java @@ -45,6 +45,9 @@ public void setup() { exchange.setProperty(SERVER_FILE_NAME, filename); exchange.setProperty(ZEEBE_VARIABLE, variables); exchange.setProperty("partyLookupFailed", partyLookupFailed); + exchange.setProperty("batchAccountLookup", + variables.get("batchAccountLookup") != null ? variables.get("batchAccountLookup") : false); + exchange.setProperty(SUB_BATCH_DETAILS, new ArrayList()); try { diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 9e67e12f..8ed37cf5 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -99,7 +99,7 @@ config: enable: true field: "payerIdentifier" splitting: - enable: false + enable: true sub-batch-size: 5 formatting: enable: false @@ -159,7 +159,7 @@ bulk_processor: hostname : "https://ph-ee-connector-bulk:8443" csv: - columnNames: "id,request_id,payment_mode,payer_identifier_type,payer_identifier,payee_identifier_type,payee_identifier,amount,currency,note,account_number,program_shortcode,cycle,payee_dfsp_id" + columnNames: "id,request_id,payment_mode,payer_identifier_type,payer_identifier,payee_identifier_type,payee_identifier,amount,currency,note,account_number,program_shortcode,cycle,payee_dfsp_id,batch_id" size : 100000 # in bytes budget-account: @@ -234,4 +234,4 @@ bpmns: batch-transactions: "bulk_processor_account_lookup-{dfspid}" - id: "lion" flows: - batch-transactions: "bulk_processor-{dfspid}" + batch-transactions: "bulk_processor-{dfspid}" \ No newline at end of file From bb517763d3a96673047b73876b13d7da77a204b3 Mon Sep 17 00:00:00 2001 From: Apurb Rajdhan <103931815+apurbraj@users.noreply.github.com> Date: Mon, 8 Jul 2024 19:05:00 +0530 Subject: [PATCH 130/156] [PHEE-697] Add Payee dfsp id header in batch transaction API (#169) * Add Payee dfsp id header in batch transaction API Sending payee dfspid variable to closedloop workflow in iniSubBatch * Addressing review comments * Adding comments * Change variable name --- .../api/definition/BatchTransactions.java | 4 +++- .../BatchTransactionsController.java | 5 ++-- .../bulk/camel/config/CamelProperties.java | 1 + .../bulk/camel/routes/InitSubBatchRoute.java | 23 +++++++++++++++++-- .../camel/routes/ProcessorStartRoute.java | 4 ++++ .../service/ProcessorStartRouteService.java | 3 +++ .../processor/bulk/zeebe/ZeebeVariables.java | 1 + 7 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/api/definition/BatchTransactions.java b/src/main/java/org/mifos/processor/bulk/api/definition/BatchTransactions.java index 720bced0..34386c38 100644 --- a/src/main/java/org/mifos/processor/bulk/api/definition/BatchTransactions.java +++ b/src/main/java/org/mifos/processor/bulk/api/definition/BatchTransactions.java @@ -5,6 +5,7 @@ import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PROGRAM_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_DFSP_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_TYPE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; @@ -24,6 +25,7 @@ String batchTransactions(HttpServletRequest httpServletRequest, HttpServletRespo @RequestHeader(value = HEADER_TYPE) String type, @RequestHeader(value = HEADER_PLATFORM_TENANT_ID) String tenant, @RequestHeader(value = HEADER_REGISTERING_INSTITUTE_ID, required = false) String registeringInstitutionId, @RequestHeader(value = HEADER_PROGRAM_ID, required = false) String programId, - @RequestHeader(value = CALLBACK, required = false) String callbackUrl) throws IOException; + @RequestHeader(value = CALLBACK, required = false) String callbackUrl, + @RequestHeader(value = PAYEE_DFSP_ID, required = false) String payeeDfspId) throws IOException; } diff --git a/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java b/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java index 90df1c72..5469ff32 100644 --- a/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java +++ b/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java @@ -3,6 +3,7 @@ import static org.mifos.processor.bulk.camel.config.CamelProperties.CALLBACK; import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PROGRAM_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.PAYEE_DFSP_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_CLIENT_CORRELATION_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_PLATFORM_TENANT_ID; @@ -67,13 +68,13 @@ public class BatchTransactionsController implements BatchTransactions { @Override public String batchTransactions(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, String requestId, String fileName, String purpose, String type, String tenant, String registeringInstitutionId, String programId, - String callbackUrl) { + String callbackUrl, String payeeDfspId) { log.info("Inside api logic"); Headers.HeaderBuilder headerBuilder = new Headers.HeaderBuilder().addHeader(HEADER_CLIENT_CORRELATION_ID, requestId) .addHeader(PURPOSE, purpose).addHeader(HEADER_TYPE, type).addHeader(HEADER_PLATFORM_TENANT_ID, tenant) .addHeader(HEADER_REGISTERING_INSTITUTE_ID, registeringInstitutionId).addHeader(HEADER_PROGRAM_ID, programId) - .addHeader(CALLBACK, callbackUrl); + .addHeader(CALLBACK, callbackUrl).addHeader(PAYEE_DFSP_ID, payeeDfspId); Optional validationResponse = isValidRequest(httpServletRequest, fileName, type); if (validationResponse.isPresent()) { diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java index 4881559d..f2339f64 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/CamelProperties.java @@ -85,4 +85,5 @@ private CamelProperties() {} public static final String ORIGINAL_TRANSACTION_LIST = "originalTransactionList"; public static final String CALLBACK = "X-CallbackURL"; public static final String CONTENT_TYPE = "Content-Type"; + public static final String PAYEE_DFSP_ID = "X-PayeeDFSP-ID"; } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index 5b876233..31bdc169 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -24,6 +24,8 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_SUB_BATCH_FAILED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ONGOING_AMOUNT; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYEE_DFSP_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYMENT_MODE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; @@ -34,6 +36,8 @@ import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.camel.Exchange; import org.apache.camel.LoggingLevel; import org.mifos.processor.bulk.config.ExternalApiPayloadConfig; @@ -68,6 +72,9 @@ public class InitSubBatchRoute extends BaseRouteBuilder { @Value("${channel.hostname}") private String channelURL; + @Value("${config.partylookup.enable}") + private boolean isPartyLookupEnabled; + @Override public void configure() throws Exception { @@ -118,6 +125,18 @@ public void configure() throws Exception { Map variables = exchange.getProperty(ZEEBE_VARIABLE, Map.class); variables.put(PAYMENT_MODE, paymentMode); variables.put(DEBULKINGDFSPID, mapping.getDebulkingDfspid() == null ? tenantName : mapping.getDebulkingDfspid()); + if (isPartyLookupEnabled && !(Boolean) variables.get(PARTY_LOOKUP_FAILED)) { + String filename = exchange.getProperty(SERVER_FILE_NAME).toString(); + String regex = ".*_sub-batch-([\\w-]+)\\.csv"; //payee DFSP Id for sub batch are extracted from the sub batch file name when party lookup is enabled and it is successful + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(filename); + + if (matcher.matches()) { + String payeeDfspId = matcher.group(1); + logger.debug("Payee DFSP Id {}", payeeDfspId); + variables.put(PAYEE_DFSP_ID, payeeDfspId); + } + } zeebeProcessStarter.startZeebeWorkflow( Utils.getBulkConnectorBpmnName(mapping.getEndpoint(), mapping.getId().toLowerCase(), tenantName), variables); exchange.setProperty(INIT_SUB_BATCH_FAILED, false); @@ -129,11 +148,11 @@ public void configure() throws Exception { Transaction transaction = transactionList.get(index); exchange.setProperty(REQUEST_ID, transaction.getRequestId()); - exchange.setProperty("payeeDFSPId", transaction.getPayeeDfspId()); + exchange.setProperty(PAYEE_DFSP_ID, transaction.getPayeeDfspId()); logger.info("REQUEST_ID: {}", transaction.getRequestId()); exchange.setProperty(TRANSACTION_LIST_ELEMENT, transaction); }).setHeader("Platform-TenantId", exchangeProperty(TENANT_NAME)) - .setHeader("X-PayeeDFSP-ID", exchangeProperty("payeeDFSPId")).to("direct:dynamic-payload-setter") + .setHeader("X-PayeeDFSP-ID", exchangeProperty(PAYEE_DFSP_ID)).to("direct:dynamic-payload-setter") .to("direct:external-api-call").to("direct:external-api-response-handler").end() // end loop block .endChoice(); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 3ea44b19..02763c84 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -23,6 +23,7 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.MERGE_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.ORDERING_ENABLED; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PARTY_LOOKUP_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYEE_DFSP_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_ENABLED; @@ -43,6 +44,7 @@ import org.apache.camel.LoggingLevel; import org.apache.tika.Tika; import org.json.JSONObject; +import org.mifos.processor.bulk.camel.config.CamelProperties; import org.mifos.processor.bulk.config.BudgetAccountConfig; import org.mifos.processor.bulk.connectors.service.ProcessorStartRouteService; import org.mifos.processor.bulk.file.FileTransferService; @@ -131,12 +133,14 @@ private void setup() { String requestId = exchange.getIn().getHeader("requestId", String.class); String purpose = exchange.getIn().getHeader("purpose", String.class); String batchId = UUID.randomUUID().toString(); + String payeeDfspId = exchange.getIn().getHeader(CamelProperties.PAYEE_DFSP_ID, String.class); String callbackUrl = exchange.getIn().getHeader("X-CallbackURL", String.class); exchange.setProperty(CALLBACK, callbackUrl); exchange.setProperty(BATCH_ID, batchId); exchange.setProperty(FILE_NAME, fileName); exchange.setProperty(REQUEST_ID, requestId); exchange.setProperty(PURPOSE, purpose); + exchange.setProperty(PAYEE_DFSP_ID, payeeDfspId); }).wireTap("direct:start-batch-process-csv"); from("direct:validate-tenant").id("direct:validate-tenant").log("Validating tenant").process(exchange -> { diff --git a/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java index b5125159..546b0ba4 100644 --- a/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java +++ b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java @@ -19,6 +19,7 @@ import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.IS_FILE_VALID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.NOTE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYEE_DFSP_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYER_IDENTIFIER_TYPE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PAYER_IDENTIFIER_VALUE; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PHASES; @@ -202,6 +203,7 @@ public void startBatchProcessCsv(Exchange exchange) throws IOException { String purpose = exchange.getProperty(PURPOSE, String.class); String batchId = exchange.getProperty(BATCH_ID, String.class); String callbackUrl = exchange.getProperty(CALLBACK, String.class); + String payeeDfspId = exchange.getProperty(PAYEE_DFSP_ID, String.class); String note = null; if (purpose == null || purpose.isEmpty()) { @@ -248,6 +250,7 @@ public void startBatchProcessCsv(Exchange exchange) throws IOException { variables.put(PAYER_IDENTIFIER_VALUE, exchange.getProperty(PAYER_IDENTIFIER_VALUE)); variables.put(REGISTERING_INSTITUTE_ID, exchange.getProperty(REGISTERING_INSTITUTE_ID)); variables.put(IS_FILE_VALID, true); + variables.put(PAYEE_DFSP_ID, payeeDfspId); processorStartRoute.setConfigProperties(variables); logger.debug("Zeebe variables published: {}", variables); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java index 83fdf99e..9736706c 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/ZeebeVariables.java @@ -165,5 +165,6 @@ private ZeebeVariables() {} public static final String PARTY_LOOKUP_SUCCESSFUL_TRANSACTION_AMOUNT = "totalApprovedAmount"; public static final String PARTY_LOOKUP_SUCCESSFUL_TRANSACTION_COUNT = "totalApprovedCount"; public static final String REGISTERING_INSTITUTION_ID = "X-Registering-Institution-ID"; + public static final String PAYEE_DFSP_ID = "payeeDfspId"; } From e958eaeb0e744427a448dae7554583b3b646af63 Mon Sep 17 00:00:00 2001 From: somanath21 <104554892+somanath21@users.noreply.github.com> Date: Mon, 15 Jul 2024 17:10:16 +0530 Subject: [PATCH 131/156] Update setup_remote_docker version for CircleCI (#154) Co-authored-by: Somanath Hugar --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f0bdc9fd..2aba252f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ jobs: steps: - checkout - setup_remote_docker: - version: 20.10.14 + version: 20.10.24 - run: name: Build and Push Docker tag Image command: | @@ -51,7 +51,7 @@ jobs: - checkout # Install Docker to build and push the image - setup_remote_docker: - version: 20.10.14 + version: 20.10.24 # Build the Docker image - run: From 1af2450a9d5132327583361012cb95878eed4ddb Mon Sep 17 00:00:00 2001 From: tdaly61 Date: Thu, 27 Mar 2025 16:07:10 +1030 Subject: [PATCH 132/156] added functional overview to top level readme --- README.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3840e736..d6d1bae5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,27 @@ -#Auto-Trigger + +# Bulk Transaction Processor +**Core Function**: Domain-specific bulk processing engine for financial operations. + +## Key Responsibilities +- Validates and transforms raw financial data (loans, repayments, fees) +- Applies business logic (interest calculations, payment allocations) +- Generates audit trails and error reports +- Prepares processed records for downstream systems + +## Inputs +- CSV/Excel files via `/api/upload` +- Direct API payloads to `/api/process` +- Database polling from staging tables + +## Outputs +- Standardized JSON to message queues (RabbitMQ/Kafka) +- Callbacks to originating systems +- Error reports in `ERROR_.csv` + +## Dependencies +- Requires `ph-ee-connector-bulk` for outbound delivery +- Integrates with accounting rule engines + ## SSL Configuration ```yaml From ddf6095ef2baae8d11c5e97bf4fce9e31fbc50e6 Mon Sep 17 00:00:00 2001 From: tdaly61 Date: Mon, 31 Mar 2025 19:50:22 +1030 Subject: [PATCH 133/156] updated build.gradle to use mifos jFrog repo and dependencies --- build.gradle | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 29d95e1b..9d1d978b 100644 --- a/build.gradle +++ b/build.gradle @@ -9,17 +9,23 @@ plugins { id 'checkstyle' id 'org.springframework.boot' version '2.6.2' id 'com.diffplug.spotless' version '6.19.0' + id "com.jfrog.artifactory" version '5.+' } - +apply plugin: "com.jfrog.artifactory" repositories { - // mavenLocal() + mavenLocal() maven { url = uri('https://repo.maven.apache.org/maven2') } maven { - url = uri('https://jfrog.sandbox.fynarfin.io/artifactory/fyn-libs-snapshot') + url = uri('https://mifos.jfrog.io/artifactory/phee-gradle-local') + } + + maven { + url = uri('https://mifos.jfrog.io/artifactory/mifosx-gradle-local') } + } apply plugin:'com.diffplug.spotless' @@ -80,7 +86,7 @@ dependencies { // miscellaneous dependency implementation 'com.google.code.gson:gson:2.8.9' implementation 'org.json:json:20210307' - implementation 'org.mifos:ph-ee-connector-common:1.8.1-SNAPSHOT' + implementation 'org.mifos:ph-ee-connector-common:1.8.1-gazelle' implementation 'org.apache.camel.springboot:camel-spring-boot-starter:3.4.0' implementation 'org.apache.camel:camel-undertow:3.4.0' implementation 'org.springframework.boot:spring-boot-starter:2.5.2' From e2d01356c22130960cb4753b67ab91e99f33dced Mon Sep 17 00:00:00 2001 From: Devarsh Date: Sun, 18 May 2025 15:23:34 +0530 Subject: [PATCH 134/156] Update CircleCI config for Java 17 --- .circleci/config.yml | 170 +++++++++++++++++++++++++++---------------- 1 file changed, 109 insertions(+), 61 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f0bdc9fd..bc1f28be 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,102 +1,150 @@ version: 2.1 +orbs: + docker-buildx: devarsh/docker-buildx-orb@0.1.1 executors: docker-executor: docker: - image: circleci/openjdk:17-buster-node-browsers-legacy - auth: - username: $DOCKERHUB_USERNAME - password: $DOCKERHUB_PASSWORD - jobs: build_and_push_tag_image: executor: docker-executor environment: JVM_OPTS: -Xmx512m TERM: dumb - GITHUB_TOKEN: ${GITHUB_TOKEN} # Add the GitHub token as an environment variable - steps: - checkout - setup_remote_docker: - version: 20.10.14 + version: 20.10.24 - run: - name: Build and Push Docker tag Image + name: Set Lowercase Docker Image Vars + # Using tr for POSIX compatibility, could use ${VAR,,} with Bash 4+ + command: | + echo "export DOCKER_ORG_LOWER=$(echo $CIRCLE_PROJECT_USERNAME | tr '[:upper:]' '[:lower:]')" >> $BASH_ENV + echo "export DOCKER_REPO_LOWER=$(echo $CIRCLE_PROJECT_REPONAME | tr '[:upper:]' '[:lower:]')" >> $BASH_ENV + echo "Using Docker Namespace: $DOCKER_ORG_LOWER" + echo "Using Docker Repository: $DOCKER_REPO_LOWER" + - docker-buildx/install + - run: + name: Check if Docker image tag exists command: | - # Set environment variables IMAGE_TAG=$CIRCLE_TAG - - # Check if the Docker image with the same tag already exists in Docker Hub - if curl -s -f -u "$DOCKERHUB_USERNAME":"$DOCKERHUB_PASSWORD" "https://hub.docker.com/v2/repositories/openmf/ph-ee-bulk-processor/tags/$IMAGE_TAG" > /dev/null; then - echo "Skipping the build and push as the tag $IMAGE_TAG already exists in Docker Hub." - exit 0 + # Using lowercase variables defined above + IMAGE_NAME="$DOCKER_ORG_LOWER/$DOCKER_REPO_LOWER" + echo "Checking for Docker image: $IMAGE_NAME:$IMAGE_TAG" + # Ensure DOCKERHUB_USERNAME and DOCKERHUB_PASSWORD are set in your CircleCI Context or Project Environment Variables + if curl -s -f -u "$DOCKERHUB_USERNAME":"$DOCKERHUB_PASSWORD" "https://hub.docker.com/v2/repositories/$IMAGE_NAME/tags/$IMAGE_TAG" > /dev/null; then + echo "Skipping the build and push as the tag $IMAGE_TAG already exists in Docker Hub for image $IMAGE_NAME." + circleci-agent step halt + else + echo "Tag $IMAGE_TAG does not exist for image $IMAGE_NAME. Proceeding with build." fi - - # Build and tag the Docker image - ./gradlew bootJar - docker build -t "openmf/ph-ee-bulk-processor:$IMAGE_TAG" . - - # Push the Docker image to Docker Hub - docker login -u "$DOCKERHUB_USERNAME" -p "$DOCKERHUB_PASSWORD" - docker push "openmf/ph-ee-bulk-processor:$IMAGE_TAG" - - # when: always # The job will be executed even if there's no match for the tag filter - - build_and_push_latest_image: + - run: + name: Build Application + command: ./gradlew bootJar + - docker-buildx/build-and-push: + # Using lowercase variables defined above + image-name: "$DOCKER_ORG_LOWER/$DOCKER_REPO_LOWER" + tag: "$CIRCLE_TAG" + # Add dockerhub credentials if needed + # dockerhub-username: "$DOCKERHUB_USERNAME" + # dockerhub-password: "$DOCKERHUB_PASSWORD" + build_and_push_branch_image: executor: docker-executor environment: JVM_OPTS: -Xmx512m TERM: dumb - steps: - checkout - # Install Docker to build and push the image - setup_remote_docker: - version: 20.10.14 - - # Build the Docker image + version: 20.10.24 - run: - name: Build Docker image + name: Set Lowercase Docker Image Vars + command: | + echo "export DOCKER_ORG_LOWER=$(echo $CIRCLE_PROJECT_USERNAME | tr '[:upper:]' '[:lower:]')" >> $BASH_ENV + echo "export DOCKER_REPO_LOWER=$(echo $CIRCLE_PROJECT_REPONAME | tr '[:upper:]' '[:lower:]')" >> $BASH_ENV + echo "Using Docker Namespace: $DOCKER_ORG_LOWER" + echo "Using Docker Repository: $DOCKER_REPO_LOWER" + - docker-buildx/install + - run: + name: Build Application command: | ./gradlew checkstyleMain - ./gradlew bootJar - docker build -t openmf/ph-ee-bulk-processor:latest . - if [ "$CIRCLE_BRANCH" != "master" ]; then - PR_NUMBER=$(basename $CIRCLE_PULL_REQUEST) - PR_TITLE=$(curl -sSL "https://api.github.com/repos/openmf/$CIRCLE_PR_REPONAME/pulls/$PR_NUMBER" | jq -r '.title') - JIRA_STORY=$(echo $PR_TITLE | cut -d "[" -f2 | cut -d "]" -f1 | tr '[A-Z]' '[a-z]') - if [ -z "$JIRA_STORY" ]; then echo "Invalid PR title" && exit 1; else echo "Ticket NO: $JIRA_STORY"; fi - docker image tag openmf/$CIRCLE_PR_REPONAME:latest openmf/$CIRCLE_PR_REPONAME:$JIRA_STORY - fi - - # Log in to DockerHub using environment variables + ./gradlew clean bootJar + - run: + name: Sanitize Branch Name + command: | + echo "export SANITIZED_BRANCH=$(echo $CIRCLE_BRANCH | sed 's/[^a-zA-Z0-9.-]/-/g' | tr '[:upper:]' '[:lower:]')" >> $BASH_ENV # Also ensure branch tag is lowercase + # First build and push with branch-latest tag + - docker-buildx/build-and-push: + image-name: "$DOCKER_ORG_LOWER/$DOCKER_REPO_LOWER" + tag: "${SANITIZED_BRANCH}-latest" + # Then build and push with branch-version tag + - docker-buildx/build-and-push: + image-name: "$DOCKER_ORG_LOWER/$DOCKER_REPO_LOWER" + tag: "${SANITIZED_BRANCH}-${CIRCLE_TAG}" + # Add dockerhub credentials if needed + # dockerhub-username: "$DOCKERHUB_USERNAME" + # dockerhub-password: "$DOCKERHUB_PASSWORD" + build_and_push_latest_image: + executor: docker-executor + environment: + JVM_OPTS: -Xmx512m + TERM: dumb + steps: + - checkout + - setup_remote_docker: + version: 20.10.24 - run: - name: Login to DockerHub - command: echo "${DOCKERHUB_PASSWORD}" | docker login -u "${DOCKERHUB_USERNAME}" --password-stdin - - # Push the Docker image to DockerHub + name: Set Lowercase Docker Image Vars + command: | + echo "export DOCKER_ORG_LOWER=$(echo $CIRCLE_PROJECT_USERNAME | tr '[:upper:]' '[:lower:]')" >> $BASH_ENV + echo "export DOCKER_REPO_LOWER=$(echo $CIRCLE_PROJECT_REPONAME | tr '[:upper:]' '[:lower:]')" >> $BASH_ENV + echo "Using Docker Namespace: $DOCKER_ORG_LOWER" + echo "Using Docker Repository: $DOCKER_REPO_LOWER" + - docker-buildx/install - run: - name: Push Docker image to DockerHub + name: Build Application command: | - if [ "$CIRCLE_BRANCH" = "master" ]; then - docker push openmf/ph-ee-bulk-processor:latest - fi - if [ "$CIRCLE_BRANCH" != "master" ]; then - PR_NUMBER=$(basename $CIRCLE_PULL_REQUEST) - PR_TITLE=$(curl -sSL "https://api.github.com/repos/openmf/$CIRCLE_PR_REPONAME/pulls/$PR_NUMBER" | jq -r '.title') - JIRA_STORY=$(echo $PR_TITLE | cut -d "[" -f2 | cut -d "]" -f1 | tr '[A-Z]' '[a-z]') - docker push openmf/$CIRCLE_PR_REPONAME:${JIRA_STORY} - fi - + ./gradlew checkstyleMain + ./gradlew clean bootJar + - docker-buildx/build-and-push: + # Using lowercase variables defined above + image-name: "$DOCKER_ORG_LOWER/$DOCKER_REPO_LOWER" + tag: "latest" + # Add dockerhub credentials if needed + # dockerhub-username: "$DOCKERHUB_USERNAME" + # dockerhub-password: "$DOCKERHUB_PASSWORD" workflows: version: 2 - build-and-push: + build-and-push-pipeline: jobs: + # Build tags matching vX.Y.Z format - build_and_push_tag_image: filters: tags: - only: /^v\d+\.\d+\.\d+$/ # Match tags in the format v1.2.3 + only: /^v\d+\.\d+\.\d+$/ + branches: + ignore: /.*/ context: - DOCKER - - build_and_push_latest_image: + # Build any branch commit (except tags) + - build_and_push_branch_image: + filters: + tags: + ignore: /.*/ + branches: + only: /.*/ context: - DOCKER + # Build 'latest' only when the branch image succeeds AND it's the main/master branch + - build_and_push_latest_image: + requires: + - build_and_push_branch_image + filters: + tags: + ignore: /.*/ + branches: + only: + - main # Or your primary branch name like 'master' + context: + - DOCKER \ No newline at end of file From b1e460958aa3b21503653a069a073de3ac46fab0 Mon Sep 17 00:00:00 2001 From: Devarsh Date: Tue, 17 Jun 2025 19:47:57 +0530 Subject: [PATCH 135/156] Update CircleCI config for Java 17 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bc1f28be..4f865ab4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,7 +81,7 @@ jobs: # Then build and push with branch-version tag - docker-buildx/build-and-push: image-name: "$DOCKER_ORG_LOWER/$DOCKER_REPO_LOWER" - tag: "${SANITIZED_BRANCH}-${CIRCLE_TAG}" + tag: "${SANITIZED_BRANCH}" # Add dockerhub credentials if needed # dockerhub-username: "$DOCKERHUB_USERNAME" # dockerhub-password: "$DOCKERHUB_PASSWORD" From e278f8d40b0e7d20f19065166355de5c182e4e53 Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Thu, 17 Jul 2025 14:35:07 +0000 Subject: [PATCH 136/156] updating to mifos jFrog --- build.gradle | 14 ++++++++--- .../BulkTransferController.java | 4 +++ .../camel/routes/BatchAggregateRoute.java | 2 +- .../camel/routes/ProcessorStartRoute.java | 3 +++ .../service/ProcessorStartRouteService.java | 6 +++-- src/main/resources/application.yaml | 25 ++++++++++--------- 6 files changed, 35 insertions(+), 19 deletions(-) diff --git a/build.gradle b/build.gradle index 29d95e1b..9d1d978b 100644 --- a/build.gradle +++ b/build.gradle @@ -9,17 +9,23 @@ plugins { id 'checkstyle' id 'org.springframework.boot' version '2.6.2' id 'com.diffplug.spotless' version '6.19.0' + id "com.jfrog.artifactory" version '5.+' } - +apply plugin: "com.jfrog.artifactory" repositories { - // mavenLocal() + mavenLocal() maven { url = uri('https://repo.maven.apache.org/maven2') } maven { - url = uri('https://jfrog.sandbox.fynarfin.io/artifactory/fyn-libs-snapshot') + url = uri('https://mifos.jfrog.io/artifactory/phee-gradle-local') + } + + maven { + url = uri('https://mifos.jfrog.io/artifactory/mifosx-gradle-local') } + } apply plugin:'com.diffplug.spotless' @@ -80,7 +86,7 @@ dependencies { // miscellaneous dependency implementation 'com.google.code.gson:gson:2.8.9' implementation 'org.json:json:20210307' - implementation 'org.mifos:ph-ee-connector-common:1.8.1-SNAPSHOT' + implementation 'org.mifos:ph-ee-connector-common:1.8.1-gazelle' implementation 'org.apache.camel.springboot:camel-spring-boot-starter:3.4.0' implementation 'org.apache.camel:camel-undertow:3.4.0' implementation 'org.springframework.boot:spring-boot-starter:2.5.2' diff --git a/src/main/java/org/mifos/processor/bulk/api/implementation/BulkTransferController.java b/src/main/java/org/mifos/processor/bulk/api/implementation/BulkTransferController.java index f0ef69fe..03457e7f 100644 --- a/src/main/java/org/mifos/processor/bulk/api/implementation/BulkTransferController.java +++ b/src/main/java/org/mifos/processor/bulk/api/implementation/BulkTransferController.java @@ -17,7 +17,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; +import lombok.extern.slf4j.Slf4j; +@Slf4j @RestController public class BulkTransferController implements BulkTransfer { @@ -36,6 +38,8 @@ public String bulkTransfer(String requestId, MultipartFile file, String fileName Headers headers = new Headers.HeaderBuilder().addHeader(HEADER_CLIENT_CORRELATION_ID, requestId).addHeader(PURPOSE, purpose) .addHeader(FILE_NAME, fileName).addHeader(HEADER_TYPE, type).addHeader(HEADER_PLATFORM_TENANT_ID, tenant).build(); Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(), headers); + log.info("Inside bulkTransfer"); + log.info("file: {}", file); fileStorageService.save(file); producerTemplate.send("direct:post-bulk-transfer", exchange); return exchange.getIn().getBody(String.class); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java index 17b92c09..4ef7d07e 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java @@ -49,7 +49,7 @@ public void configure() throws Exception { .setHeader(HEADER_PLATFORM_TENANT_ID, simple("${exchangeProperty." + TENANT_ID + "}")).process(exchange -> { logger.info(exchange.getIn().getHeaders().toString()); }).toD(operationsAppConfig.batchAggregateUrl + "${exchangeProperty." + BATCH_ID + "}?bridgeEndpoint=true") - .log(LoggingLevel.DEBUG, "Batch aggregate API response: \n\n ${body}") + .log(LoggingLevel.INFO, "Batch aggregate API response: \n\n ${body}") .log(LoggingLevel.INFO, "Aggregate Response body: ${body}"); from("direct:batch-aggregate-response-handler").id("direct:batch-aggregate-response-handler") diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index 3ea44b19..d7c8795b 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -256,6 +256,9 @@ public boolean verifyData(File file) throws IOException { public String getWorkflowForTenant(String tenantId, String useCase) { for (TenantImplementation tenant : tenantImplementationProperties.getTenants()) { + logger.info("FRED1 Tenant id: {} ", tenant.getId()); + logger.info("FRED1 tenandId {} ", tenantId); + logger.info("FRED1 useCase {} ", useCase); if (tenant.getId().equals(tenantId)) { return tenant.getFlows().getOrDefault(useCase, "default"); } diff --git a/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java index b5125159..52be7c49 100644 --- a/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java +++ b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java @@ -250,14 +250,16 @@ public void startBatchProcessCsv(Exchange exchange) throws IOException { variables.put(IS_FILE_VALID, true); processorStartRoute.setConfigProperties(variables); - logger.debug("Zeebe variables published: {}", variables); - logger.debug("Variables published to zeebe: {}", variables); + logger.info("Zeebe variables published: {}", variables); + logger.info("Variables published to zeebe: {}", variables); JSONObject response = new JSONObject(); String bpmn = processorStartRoute.getWorkflowForTenant(exchange.getProperty(TENANT_NAME).toString(), "batch-transactions"); try { String tenantSpecificWorkflowId = bpmn.replace("{dfspid}", exchange.getProperty(TENANT_NAME).toString()); + logger.info("Tenant specific workflow id: {}", tenantSpecificWorkflowId); + logger.info("FRED: tenant is < {} > ", exchange.getProperty(TENANT_NAME).toString()); String txnId = zeebeProcessStarter.startZeebeWorkflow(tenantSpecificWorkflowId, "", variables); if (txnId == null || txnId.isEmpty()) { response.put("errorCode", 500); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index e3b6a04d..d3e9dd4f 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -45,7 +45,7 @@ mock-payment-schema: authorization: "/batches/" channel: - hostname: "https://ph-ee-connector-channel:8443" + hostname: "http://ph-ee-connector-channel:82" cloud: aws: @@ -65,7 +65,7 @@ cloud: connection-string: -tenants: "ibank-usa,ibank-india" +tenants: "greenbank, bluebank" bpmn: flows: @@ -156,7 +156,7 @@ identity_account_mapper: batch_account_lookup_callback: /batchAccountLookupCallback bulk_processor: - hostname : "https://ph-ee-connector-bulk:8443" + hostname : "http://ph-ee-connector-bulk:82" csv: columnNames: "id,request_id,payment_mode,payer_identifier_type,payer_identifier,payee_identifier_type,payee_identifier,amount,currency,note,account_number,program_shortcode,cycle,payee_dfsp_id" @@ -195,7 +195,7 @@ payment-mode: - id: "CLOSEDLOOP" type: "BULK" endpoint: "bulk_connector_{MODE}-{dfspid}" - debulkingDfspid: "lion" + debulkingDfspid: "greenbank" batch-authorization: callback-url: "https://ph-ee-connector-bulk:8443/authorization/callback" @@ -221,15 +221,16 @@ gov-stack-client: bpmns: tenants: - - id: "gorilla" + - id: "greenbank" flows: + payment-transfer: "minimal_mock_fund_transfer-{dfspid}" batch-transactions: "bulk_processor-{dfspid}" - - id: "wakanda" - flows: - batch-transactions: "bulk_processor-{dfspid}" - - id: "rhino" - flows: - batch-transactions: "bulk_processor_account_lookup-{dfspid}" - - id: "lion" + - id: "bluebank" flows: batch-transactions: "bulk_processor-{dfspid}" + # - id: "rhino" + # flows: + # batch-transactions: "bulk_processor_account_lookup-{dfspid}" + # - id: "lion" + # flows: + # batch-transactions: "bulk_processor-{dfspid}" From 54b8429a57d16ad886cf76da726110a448d15adb Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Thu, 17 Jul 2025 14:41:20 +0000 Subject: [PATCH 137/156] changed version / boot jar to gazelle-1.1.0 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 9d1d978b..c2625ab0 100644 --- a/build.gradle +++ b/build.gradle @@ -225,7 +225,7 @@ configure(this) { group = 'org.mifos' -version = '0.0.1-SNAPSHOT' +version = 'gazelle-1.1.0' description = 'ph-ee-processor-bulk' sourceCompatibility = '17' From fb58df3f504d05c7b85a105175f094e4d8593a04 Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Thu, 17 Jul 2025 15:38:55 +0000 Subject: [PATCH 138/156] fixing style issues --- .circleci/config.yml | 2 +- build.gradle | 3 +-- .../bulk/api/implementation/BulkTransferController.java | 2 +- .../bulk/connectors/service/ProcessorStartRouteService.java | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4f865ab4..930d5efa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -147,4 +147,4 @@ workflows: only: - main # Or your primary branch name like 'master' context: - - DOCKER \ No newline at end of file + - DOCKER diff --git a/build.gradle b/build.gradle index c2625ab0..e0c57f05 100644 --- a/build.gradle +++ b/build.gradle @@ -21,11 +21,10 @@ repositories { maven { url = uri('https://mifos.jfrog.io/artifactory/phee-gradle-local') } - + maven { url = uri('https://mifos.jfrog.io/artifactory/mifosx-gradle-local') } - } apply plugin:'com.diffplug.spotless' diff --git a/src/main/java/org/mifos/processor/bulk/api/implementation/BulkTransferController.java b/src/main/java/org/mifos/processor/bulk/api/implementation/BulkTransferController.java index 03457e7f..8a8d1e6e 100644 --- a/src/main/java/org/mifos/processor/bulk/api/implementation/BulkTransferController.java +++ b/src/main/java/org/mifos/processor/bulk/api/implementation/BulkTransferController.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; +import lombok.extern.slf4j.Slf4j; import org.apache.camel.Exchange; import org.apache.camel.ProducerTemplate; import org.mifos.processor.bulk.api.definition.BulkTransfer; @@ -17,7 +18,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; -import lombok.extern.slf4j.Slf4j; @Slf4j @RestController diff --git a/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java index 52be7c49..3fedc201 100644 --- a/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java +++ b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java @@ -259,7 +259,7 @@ public void startBatchProcessCsv(Exchange exchange) throws IOException { try { String tenantSpecificWorkflowId = bpmn.replace("{dfspid}", exchange.getProperty(TENANT_NAME).toString()); logger.info("Tenant specific workflow id: {}", tenantSpecificWorkflowId); - logger.info("FRED: tenant is < {} > ", exchange.getProperty(TENANT_NAME).toString()); + logger.info("FRED: tenant is < {} > ", exchange.getProperty(TENANT_NAME).toString()); String txnId = zeebeProcessStarter.startZeebeWorkflow(tenantSpecificWorkflowId, "", variables); if (txnId == null || txnId.isEmpty()) { response.put("errorCode", 500); From 7356c0680c7fa110c1d5a66d8f2b36a9c9ed38a0 Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Thu, 20 Nov 2025 23:50:03 +0000 Subject: [PATCH 139/156] moved bulk to port 8080, http and added config PreFlighCorsFilter to solve cors issues, not sure about the move to 8080 --- .../camel/routes/ProcessorStartRoute.java | 1 + .../processor/bulk/config/CorsProperties.java | 21 +++ .../bulk/config/PreflightCorsFilter.java | 58 +++++++ .../service/ProcessorStartRouteService.java | 2 + .../bulk/zeebe/worker/InitSubBatchWorker.java | 147 ++++++++++-------- src/main/resources/application.yaml | 79 ++++++---- 6 files changed, 216 insertions(+), 92 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/config/CorsProperties.java create mode 100644 src/main/java/org/mifos/processor/bulk/config/PreflightCorsFilter.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index d7c8795b..32b81b8e 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -243,6 +243,7 @@ public boolean verifyData(File file) throws IOException { while ((line = br.readLine()) != null) { String[] row = line.split(","); if (row.length != columnNames.size()) { + logger.debug("DEBUG FRED10 Row invalid {} {}", row.length, columnNames.size()); logger.debug("Row invalid {} {}", row.length, columnNames.size()); return false; } diff --git a/src/main/java/org/mifos/processor/bulk/config/CorsProperties.java b/src/main/java/org/mifos/processor/bulk/config/CorsProperties.java new file mode 100644 index 00000000..1741bff8 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/config/CorsProperties.java @@ -0,0 +1,21 @@ +package org.mifos.processor.bulk.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +import java.util.List; + +@Configuration +@ConfigurationProperties(prefix = "cors") +public class CorsProperties { + + private List allowedOrigins; + + public List getAllowedOrigins() { + return allowedOrigins; + } + + public void setAllowedOrigins(List allowedOrigins) { + this.allowedOrigins = allowedOrigins; + } +} diff --git a/src/main/java/org/mifos/processor/bulk/config/PreflightCorsFilter.java b/src/main/java/org/mifos/processor/bulk/config/PreflightCorsFilter.java new file mode 100644 index 00000000..f7b21354 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/config/PreflightCorsFilter.java @@ -0,0 +1,58 @@ +package org.mifos.processor.bulk.config; + +import org.springframework.stereotype.Component; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.List; + +@Component +public class PreflightCorsFilter implements Filter { + + private final List allowedOrigins; + + @Autowired + public PreflightCorsFilter(CorsProperties corsProperties) { + this.allowedOrigins = corsProperties.getAllowedOrigins(); + } + + @Override + public void doFilter(javax.servlet.ServletRequest servletRequest, + javax.servlet.ServletResponse servletResponse, + FilterChain filterChain) throws IOException, ServletException { + + HttpServletRequest request = (HttpServletRequest) servletRequest; + HttpServletResponse response = (HttpServletResponse) servletResponse; + + String origin = request.getHeader("Origin"); + + if ("OPTIONS".equalsIgnoreCase(request.getMethod()) && origin != null && allowedOrigins.contains(origin)) { + response.setStatus(HttpServletResponse.SC_OK); + response.setHeader("Access-Control-Allow-Origin", origin); + response.setHeader("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS"); + response.setHeader("Access-Control-Allow-Headers", + "content-type,platform-tenantid,purpose,type,x-callback-url,x-correlationid,x-program-id,x-registering-institution-id,x-signature"); + response.setHeader("Access-Control-Allow-Credentials", "true"); + response.setHeader("Access-Control-Max-Age", "86400"); + return; // short-circuit OPTIONS preflight + } + + filterChain.doFilter(servletRequest, servletResponse); + } + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + + } + + @Override + public void destroy() { + } + +} diff --git a/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java index 3fedc201..1c2ecbdb 100644 --- a/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java +++ b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java @@ -257,6 +257,8 @@ public void startBatchProcessCsv(Exchange exchange) throws IOException { String bpmn = processorStartRoute.getWorkflowForTenant(exchange.getProperty(TENANT_NAME).toString(), "batch-transactions"); try { + logger.info("FREDa "); + logger.info("FREDa: tenant is < {} > ", exchange.getProperty(TENANT_NAME).toString()); String tenantSpecificWorkflowId = bpmn.replace("{dfspid}", exchange.getProperty(TENANT_NAME).toString()); logger.info("Tenant specific workflow id: {}", tenantSpecificWorkflowId); logger.info("FRED: tenant is < {} > ", exchange.getProperty(TENANT_NAME).toString()); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java index 2acacf11..e35c2382 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java @@ -1,27 +1,15 @@ package org.mifos.processor.bulk.zeebe.worker; -import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; -import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_DETAILS; -import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_ENTITY; -import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; -import static org.mifos.processor.bulk.camel.config.CamelProperties.ZEEBE_VARIABLE; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_FAILURE_SUB_BATCHES; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_SUB_BATCH_FAILED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_SUCCESS_SUB_BATCHES; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REMAINING_SUB_BATCH; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_ENABLED; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SUB_BATCHES; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; +import static org.mifos.processor.bulk.camel.config.CamelProperties.*; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import org.apache.camel.Exchange; import org.apache.camel.support.DefaultExchange; import org.mifos.processor.bulk.camel.routes.RouteId; @@ -35,55 +23,88 @@ public class InitSubBatchWorker extends BaseWorker { @Autowired private ObjectMapper objectMapper; + // FIXED: Generic-safe, null-safe, no inference issues + private static List toStringList(Object obj) { + if (obj == null) { + return new ArrayList<>(); + } + if (obj instanceof List list) { + List result = new ArrayList<>(); + for (Object item : list) { + if (item != null) { + result.add(item.toString()); + } + } + return result; + } + return new ArrayList<>(); + } + + // FIXED: For SUB_BATCH_DETAILS → List expected + private static List toObjectList(Object obj) { + if (obj == null) { + return new ArrayList<>(); + } + if (obj instanceof List list) { + return new ArrayList<>(list); // safe: List → List + } + return new ArrayList<>(); + } + @Override public void setup() { - - /** - * Starts the new worker for initialising sub batches. Performs below tasks 1. Downloads the file from cloud. 2. - * Parse the data into POJO. 3. Initiates workflow based on the payment_mode - */ newWorker(Worker.INIT_SUB_BATCH, (client, job) -> { logger.info("Started INIT_SUB_BATCH worker"); - logger.debug("Job '{}' started from process '{}' with key {}", job.getType(), job.getBpmnProcessId(), job.getKey()); + Map variables = job.getVariablesAsMap(); - List subBatches = (List) variables.get(SUB_BATCHES); - if (subBatches == null) { - subBatches = new ArrayList<>(); + // 100% SAFE LISTS — NO NPE, NO COMPILATION ISSUES + List subBatches = toStringList(variables.get(SUB_BATCHES)); + List successSubBatches = toStringList(variables.get(INIT_SUCCESS_SUB_BATCHES)); + List failureSubBatches = toStringList(variables.get(INIT_FAILURE_SUB_BATCHES)); + List subBatchDetails = toObjectList(variables.get(SUB_BATCH_DETAILS)); + + // Early exit + if (subBatches.isEmpty()) { + logger.info("No sub-batches to process. Completing job early."); + variables.put(REMAINING_SUB_BATCH, 0); + variables.put(SUB_BATCHES, new ArrayList()); + variables.put(INIT_SUCCESS_SUB_BATCHES, new ArrayList()); + variables.put(INIT_FAILURE_SUB_BATCHES, new ArrayList()); + client.newCompleteCommand(job.getKey()) + .variables(variables) + .send() + .join(); + return; } - List successSubBatches = (List) variables.get(INIT_SUCCESS_SUB_BATCHES); - if (successSubBatches == null) { - successSubBatches = new ArrayList<>(); - } - List failureSubBatches = (List) variables.get(INIT_FAILURE_SUB_BATCHES); - if (failureSubBatches == null) { - failureSubBatches = new ArrayList<>(); - } - boolean isSplittingEnabled = (boolean) variables.get(SPLITTING_ENABLED); - if (!isSplittingEnabled) { - subBatches.add((String) variables.get(FILE_NAME)); + // Handle non-splitting mode + Boolean splittingEnabled = (Boolean) variables.get(SPLITTING_ENABLED); + if (Boolean.FALSE.equals(splittingEnabled)) { + String fileName = (String) variables.get(FILE_NAME); + if (fileName != null && !subBatches.contains(fileName)) { + subBatches.add(fileName); + } } - List subBatchObjectList = (List) variables.get(SUB_BATCH_DETAILS); - logger.debug("Subbatch entity list in init sub batch worker: {}", subBatchObjectList); - - List subBatchEntityList = objectMapper.convertValue(subBatchObjectList, new TypeReference<>() {}); + // Safe remove + String currentFile = subBatches.remove(0); - String fileName = subBatches.remove(0); - SubBatchEntity subBatchEntity = null; + // Parse sub-batch details + List subBatchEntityList = objectMapper.convertValue( + subBatchDetails, + new TypeReference>() {} + ); - for (SubBatchEntity subBatch : subBatchEntityList) { - if (subBatch.getRequestFile().contains(fileName)) { - subBatchEntity = subBatch; - logger.info("SubBatchEntity found"); - } - } - logger.debug("BatchEntity for this subbatch is {}", objectMapper.writeValueAsString(subBatchEntity)); + SubBatchEntity subBatchEntity = subBatchEntityList.stream() + .filter(e -> e.getRequestFile() != null && e.getRequestFile().contains(currentFile)) + .findFirst() + .orElse(null); + // Setup Camel exchange Exchange exchange = new DefaultExchange(camelContext); exchange.setProperty(TENANT_NAME, variables.get(TENANT_ID)); - exchange.setProperty(SERVER_FILE_NAME, fileName); + exchange.setProperty(SERVER_FILE_NAME, currentFile); exchange.setProperty(BATCH_ID, variables.get(BATCH_ID)); exchange.setProperty(REQUEST_ID, variables.get(REQUEST_ID)); exchange.setProperty(PURPOSE, variables.get(PURPOSE)); @@ -92,21 +113,25 @@ public void setup() { sendToCamelRoute(RouteId.INIT_SUB_BATCH, exchange); - Boolean subBatchFailed = exchange.getProperty(INIT_SUB_BATCH_FAILED, Boolean.class); - if (subBatchFailed != null && subBatchFailed) { - failureSubBatches.add(fileName); + Boolean failed = exchange.getProperty(INIT_SUB_BATCH_FAILED, Boolean.class); + if (Boolean.TRUE.equals(failed)) { + failureSubBatches.add(currentFile); } else { - successSubBatches.add(fileName); + successSubBatches.add(currentFile); } + // Update Zeebe variables variables.put(REMAINING_SUB_BATCH, subBatches.size()); - variables.put(SUB_BATCHES, subBatches); - variables.put(INIT_SUCCESS_SUB_BATCHES, successSubBatches); - variables.put(INIT_FAILURE_SUB_BATCHES, failureSubBatches); + variables.put(SUB_BATCHES, new ArrayList<>(subBatches)); + variables.put(INIT_SUCCESS_SUB_BATCHES, new ArrayList<>(successSubBatches)); + variables.put(INIT_FAILURE_SUB_BATCHES, new ArrayList<>(failureSubBatches)); - client.newCompleteCommand(job.getKey()).variables(variables).send(); - logger.info("Completed INIT_SUB_BATCH worker. Remaining subbatches {}", subBatches.size()); + client.newCompleteCommand(job.getKey()) + .variables(variables) + .send() + .join(); + + logger.info("Completed INIT_SUB_BATCH worker. Remaining sub-batches: {}", subBatches.size()); }); } - -} +} \ No newline at end of file diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index d3e9dd4f..efd795b9 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -1,11 +1,22 @@ +# camel: +# server-port: 5000 +# disable-ssl: false +# springboot: +# main-run-controller: true +# dataformat: +# json-jackson: +# auto-discover-object-mapper: true + +# TD Nov 2025 -- terminate TLS at NGINX run pod on http => solve cors issues from operations-web camel: - server-port: 5000 - disable-ssl: false - springboot: - main-run-controller: true - dataformat: - json-jackson: - auto-discover-object-mapper: true + server-port: 0 + disable-ssl: true + +cors: + allowed-origins: + - "http://ops.mifos.gazelle.localhost" + - "https://ops.mifos.gazelle.test" + kafka: bootstrapAddress: "kafka:9092" @@ -30,7 +41,7 @@ zeebe: contactpoint: "zeebe-zeebe-gateway:26500" operations-app: - contactpoint: "https://ops-bk.sandbox.mifos.io" + contactpoint: "https://ops-bk.mifos.gazelle.localhost" username: "mifos" password: "password" endpoints: @@ -67,21 +78,7 @@ cloud: tenants: "greenbank, bluebank" -bpmn: - flows: - payment-transfer: "PayerFundTransfer-{dfspid}" - transaction-request: "PayeeTransactionRequest-{dfspid}" - party-registration: "PartyRegistration-{dfspid}" - gsma-base-transaction: "gsma_base_transaction-{dfspid}" - gsma-int-transfer: "gsma_int_transfer-{dfspid}" - gsma-payee-process: "gsma_payee_process" - gsma-bill-payment: "gsma_bill_payment" - gsma-link-based-payment: "gsma_link_transfer" - international-remittance-payee: "international_remittance_payee_process-{dfspid}" - international-remittance-payer: "international_remittance_payer_process-{dfspid}" - debit-party-process: "debit_party_process-{dfspid}" - bulk-processor: "bulk_processor-{dfspid}" - slcb: "slcb-{dfspid}" + #payment mode moved to a separate application property file for helm rewrite @@ -133,13 +130,13 @@ callback-phases: - 100 server: - ssl: - key-alias: "tomcat-https" - key-store: "classpath:keystore.jks" - key-store-type: JKS - key-password: "password" - key-store-password: "password" - port: 8443 + # ssl: + # key-alias: "tomcat-https" + # key-store: "classpath:keystore.jks" + # key-store-type: JKS + # key-password: "password" + # key-store-password: "password" + port: 8080 security: @@ -196,8 +193,11 @@ payment-mode: type: "BULK" endpoint: "bulk_connector_{MODE}-{dfspid}" debulkingDfspid: "greenbank" + batch-authorization: - callback-url: "https://ph-ee-connector-bulk:8443/authorization/callback" + callback-url: "${BULK_PROCESSOR_CALLBACK_URL:https://bulk-processor.mifos.gazelle.test/authorization/callback}" + + pubsub: room: @@ -219,12 +219,29 @@ gov-stack-client: header-value: "PAYMENT-BB" +bpmn: + flows: + payment-transfer: "PayerFundTransfer-{dfspid}" + transaction-request: "PayeeTransactionRequest-{dfspid}" + party-registration: "PartyRegistration-{dfspid}" + gsma-base-transaction: "gsma_base_transaction-{dfspid}" + gsma-int-transfer: "gsma_int_transfer-{dfspid}" + gsma-payee-process: "gsma_payee_process" + gsma-bill-payment: "gsma_bill_payment" + gsma-link-based-payment: "gsma_link_transfer" + international-remittance-payee: "international_remittance_payee_process-{dfspid}" + international-remittance-payer: "international_remittance_payer_process-{dfspid}" + debit-party-process: "debit_party_process-{dfspid}" + bulk-processor: "bulk_processor-{dfspid}" + slcb: "slcb-{dfspid}" + bpmns: tenants: - id: "greenbank" flows: payment-transfer: "minimal_mock_fund_transfer-{dfspid}" batch-transactions: "bulk_processor-{dfspid}" + fred-transactions: "bulk_processor_account_lookup-{dfspid}" - id: "bluebank" flows: batch-transactions: "bulk_processor-{dfspid}" From 23bdecdb5542d2ad639b2a6a4a4d14ba87c041b0 Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Fri, 21 Nov 2025 00:25:26 +0000 Subject: [PATCH 140/156] missed application.xml changes in previous commit --- src/main/resources/application.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index a57e1e95..fe5b9953 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -194,12 +194,8 @@ payment-mode: - id: "CLOSEDLOOP" type: "BULK" endpoint: "bulk_connector_{MODE}-{dfspid}" -<<<<<<< HEAD debulkingDfspid: "greenbank" -======= - debulkingDfspid: "wakanda" ->>>>>>> master batch-authorization: callback-url: "${BULK_PROCESSOR_CALLBACK_URL:https://bulk-processor.mifos.gazelle.test/authorization/callback}" @@ -250,7 +246,6 @@ bpmns: fred-transactions: "bulk_processor_account_lookup-{dfspid}" - id: "bluebank" flows: -<<<<<<< HEAD batch-transactions: "bulk_processor-{dfspid}" # - id: "rhino" # flows: @@ -258,6 +253,3 @@ bpmns: # - id: "lion" # flows: # batch-transactions: "bulk_processor-{dfspid}" -======= - batch-transactions: "bulk_processor-{dfspid}" ->>>>>>> master From a80d1fca4f3249951a067467a3aa7ca565a1c7f1 Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Fri, 21 Nov 2025 09:56:38 +0000 Subject: [PATCH 141/156] compile time fixes subsequent to merge --- .../HttpClientConfigurerTrustAllCACerts.java | 7 +- .../routes/AccountLookupCallbackRoute.java | 1 + .../bulk/camel/routes/AccountLookupRoute.java | 1 + .../bulk/camel/routes/DeDuplicationRoute.java | 1 + .../camel/routes/FileProcessingRoute.java | 1 + .../bulk/camel/routes/FormattingRoute.java | 1 + .../bulk/camel/routes/InitSubBatchRoute.java | 3 +- .../bulk/camel/routes/MergeBackRoute.java | 80 +++++++++++++------ .../camel/routes/OperationsAuthRoute.java | 2 + .../bulk/camel/routes/OrderingRoute.java | 1 + .../bulk/camel/routes/SendCallbackRoute.java | 2 + .../bulk/camel/routes/SplittingRoute.java | 7 +- .../service/ProcessorStartRouteService.java | 59 +++++++++----- .../bulk/service/BatchAccountLookup.java | 31 ++++--- .../service/FileProcessingRouteService.java | 2 + .../bulk/zeebe/worker/InitSubBatchWorker.java | 1 - 16 files changed, 139 insertions(+), 61 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java b/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java index 7a0a6b89..c2064227 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java @@ -13,6 +13,7 @@ import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.conn.ssl.TrustStrategy; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; @@ -41,11 +42,11 @@ public boolean isTrusted(X509Certificate[] arg0, String arg1) throws Certificate } catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) { logger.debug(e.getMessage()); } - clientBuilder.setSslcontext(sslContext); - + clientBuilder.setSSLContext(sslContext); // don't check Hostnames, either. // -- use SSLConnectionSocketFactory.getDefaultHostnameVerifier(), if you don't want to weaken - HostnameVerifier hostnameVerifier = SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER; + HostnameVerifier hostnameVerifier = NoopHostnameVerifier.INSTANCE; + // TD Deprecated HostnameVerifier hostnameVerifier = SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER; // here's the special part: // -- need to create an SSL Socket Factory, to use our weakened "trust strategy"; diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupCallbackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupCallbackRoute.java index 216d051c..9b4a3c28 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupCallbackRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupCallbackRoute.java @@ -33,6 +33,7 @@ public class AccountLookupCallbackRoute extends BaseRouteBuilder { private Integer totalApprovedCount; @Override + @SuppressWarnings("unchecked") public void configure() throws Exception { from("direct:accountLookupCallback").id("direct:accountLookupCallback") .log("Starting route " + RouteId.ACCOUNT_LOOKUP_CALLBACK.name()).to("direct:download-file") diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java index e40a0c48..79ffce9f 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java @@ -26,6 +26,7 @@ import org.springframework.stereotype.Component; @Component +@SuppressWarnings("unchecked") public class AccountLookupRoute extends BaseRouteBuilder { @Value("${identity_account_mapper.account_lookup}") diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/DeDuplicationRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/DeDuplicationRoute.java index 032fe09d..c4c640c3 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/DeDuplicationRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/DeDuplicationRoute.java @@ -24,6 +24,7 @@ public class DeDuplicationRoute extends BaseRouteBuilder { @Override + @SuppressWarnings("unchecked") public void configure() throws Exception { from(RouteId.DE_DUPLICATION.getValue()).id(RouteId.DE_DUPLICATION.getValue()) .log("Started route " + RouteId.DE_DUPLICATION.getValue()).to("direct:download-file").to("direct:get-transaction-array") diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java index 74cbb943..cf40f128 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java @@ -31,6 +31,7 @@ public class FileProcessingRoute extends BaseRouteBuilder { private CsvMapper csvMapper; @Override + @SuppressWarnings("unchecked") public void configure() { /** diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java index f88d8f62..0589471d 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FormattingRoute.java @@ -25,6 +25,7 @@ public class FormattingRoute extends BaseRouteBuilder { private Standard formattingStandard; @Override + @SuppressWarnings("unchecked") public void configure() throws Exception { // parsing enum from application.yaml string formattingStandard = Standard.valueOf(standard); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index 31bdc169..7c8be8c8 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -76,6 +76,7 @@ public class InitSubBatchRoute extends BaseRouteBuilder { private boolean isPartyLookupEnabled; @Override + @SuppressWarnings("unchecked") public void configure() throws Exception { /** @@ -133,7 +134,7 @@ public void configure() throws Exception { if (matcher.matches()) { String payeeDfspId = matcher.group(1); - logger.debug("Payee DFSP Id {}", payeeDfspId); + logger.info("Payee DFSP Id {}", payeeDfspId); variables.put(PAYEE_DFSP_ID, payeeDfspId); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java index db6d6456..44cb4f50 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java @@ -24,22 +24,38 @@ public class MergeBackRoute extends BaseRouteBuilder { public void configure() throws Exception { /** - * Base route for kicking off merge back logic. Performs below tasks. 1. Picks the top two files from the array - * of files to be merged. 2. Merges them into single CSV. 3. Uploads the CSV to S3. 4. Updated the exchange - * variables. + * Base route for kicking off merge back logic. Performs below tasks: + * 1. Picks the top two files from the array of files to be merged. + * 2. Merges them into single CSV. + * 3. Uploads the CSV to S3. + * 4. Updates the exchange variables. */ - from(RouteId.MERGE_BACK.getValue()).id(RouteId.MERGE_BACK.getValue()).log("Starting route " + RouteId.MERGE_BACK.name()).choice() - .when(exchange -> exchange.getProperty(MERGE_FILE_LIST, List.class).size() == 0).log("Nothing to merge") - .setProperty(MERGE_FAILED, constant(false)).setProperty(MERGE_COMPLETED, constant(true)) - .when(exchange -> exchange.getProperty(MERGE_FILE_LIST, List.class).size() == 1).process(exchange -> { - exchange.setProperty(MERGE_FAILED, false); - exchange.setProperty(MERGE_COMPLETED, true); - String resultFile = (String) exchange.getProperty(MERGE_FILE_LIST, List.class).get(0); - setResultFileProperty(exchange, resultFile); - }).otherwise().to("direct:start-merge").endChoice(); + from(RouteId.MERGE_BACK.getValue()) + .id(RouteId.MERGE_BACK.getValue()) + .log("Starting route " + RouteId.MERGE_BACK.name()) + .choice() + .when(exchange -> exchange.getProperty(MERGE_FILE_LIST, List.class).size() == 0) + .log("Nothing to merge") + .setProperty(MERGE_FAILED, constant(false)) + .setProperty(MERGE_COMPLETED, constant(true)) + .when(exchange -> exchange.getProperty(MERGE_FILE_LIST, List.class).size() == 1) + .process(exchange -> { + exchange.setProperty(MERGE_FAILED, false); + exchange.setProperty(MERGE_COMPLETED, true); + @SuppressWarnings("unchecked") + List mergeFileList = (List) exchange.getProperty(MERGE_FILE_LIST, List.class); + String resultFile = mergeFileList.get(0); + setResultFileProperty(exchange, resultFile); + }) + .otherwise() + .to("direct:start-merge") + .endChoice(); // starts the merge process, merges the file and uploads the file in s3 - from("direct:start-merge").id("direct:start-merge").log("Starting route direct:start-merge").to("direct:download-file-to-be-merged") + from("direct:start-merge") + .id("direct:start-merge") + .log("Starting route direct:start-merge") + .to("direct:download-file-to-be-merged") .process(exchange -> { String file1 = exchange.getProperty(FILE_1, String.class); String file2 = exchange.getProperty(FILE_2, String.class); @@ -51,15 +67,19 @@ public void configure() throws Exception { } if (exchange.getProperty(MERGE_ITERATION, Integer.class) == 1) { // generate new name for merged file in case of first iteration - String newFileName = System.currentTimeMillis() + "_" + exchange.getProperty(BATCH_ID, String.class) + ".csv"; + String newFileName = System.currentTimeMillis() + "_" + + exchange.getProperty(BATCH_ID, String.class) + ".csv"; new File(mergedFile).renameTo(new File(newFileName)); exchange.setProperty(LOCAL_FILE_PATH, newFileName); } else { exchange.setProperty(LOCAL_FILE_PATH, mergedFile); } - }).to("direct:upload-file").process(exchange -> { + }) + .to("direct:upload-file") + .process(exchange -> { String mergedFileServerName = exchange.getProperty(SERVER_FILE_NAME, String.class); - List mergeList = exchange.getProperty(MERGE_FILE_LIST, List.class); + @SuppressWarnings("unchecked") + List mergeList = (List) exchange.getProperty(MERGE_FILE_LIST, List.class); String first = mergeList.remove(0); String second = mergeList.remove(0); logger.info("Merge iteration {}, for list, {}", exchange.getProperty(MERGE_ITERATION), mergeList); @@ -76,21 +96,29 @@ public void configure() throws Exception { exchange.setProperty(MERGE_FILE_LIST, mergeList); - // make sures to remove the files from local storage + // make sure to remove the files from local storage new File(exchange.getProperty(FILE_1, String.class)).delete(); new File(exchange.getProperty(FILE_2, String.class)).delete(); }); - // downloads the two files(using FIFO access pattern) from s3 which is to be merged. - from("direct:download-file-to-be-merged").id("direct:download-file-to-be-merged") - .log("Starting route direct:download-file-to-be-merged").log("Downloading files to be merged").process(exchange -> { - List mergeList = exchange.getProperty(MERGE_FILE_LIST, List.class); + // downloads the two files (using FIFO access pattern) from s3 which are to be merged + from("direct:download-file-to-be-merged") + .id("direct:download-file-to-be-merged") + .log("Starting route direct:download-file-to-be-merged") + .log("Downloading files to be merged") + .process(exchange -> { + @SuppressWarnings("unchecked") + List mergeList = (List) exchange.getProperty(MERGE_FILE_LIST, List.class); exchange.setProperty(SERVER_FILE_NAME, mergeList.get(0)); - }).to("direct:download-file") // downloading first file - .setProperty(FILE_1, exchangeProperty(LOCAL_FILE_PATH)).process(exchange -> { - List mergeList = exchange.getProperty(MERGE_FILE_LIST, List.class); + }) + .to("direct:download-file") // downloading first file + .setProperty(FILE_1, exchangeProperty(LOCAL_FILE_PATH)) + .process(exchange -> { + @SuppressWarnings("unchecked") + List mergeList = (List) exchange.getProperty(MERGE_FILE_LIST, List.class); exchange.setProperty(SERVER_FILE_NAME, mergeList.get(1)); - }).to("direct:download-file") // downloading second file + }) + .to("direct:download-file") // downloading second file .setProperty(FILE_2, exchangeProperty(LOCAL_FILE_PATH)); } @@ -98,4 +126,4 @@ public void configure() throws Exception { public void setResultFileProperty(Exchange exchange, String fileName) { exchange.setProperty(RESULT_FILE, Utils.getAwsFileUrl(awsS3BaseUrl, fileName)); } -} +} \ No newline at end of file diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/OperationsAuthRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/OperationsAuthRoute.java index da073cc9..35f2c5f9 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/OperationsAuthRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/OperationsAuthRoute.java @@ -15,6 +15,7 @@ public class OperationsAuthRoute extends BaseRouteBuilder { @Override + @SuppressWarnings("unchecked") public void configure() throws Exception { from("rest:get:test/auth").to("direct:get-access-token"); @@ -29,6 +30,7 @@ public void configure() throws Exception { /** * Save Access Token to AccessTokenStore */ + from("direct:access-token-save").id("access-token-save").unmarshal().json(JsonLibrary.Jackson, HashMap.class).process(exchange -> { // TODO: Figure out access token storage if required Map jsonObject = exchange.getIn().getBody(HashMap.class); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java index 5f23e11c..e0665fa7 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/OrderingRoute.java @@ -17,6 +17,7 @@ public class OrderingRoute extends BaseRouteBuilder { private String orderingField; @Override + @SuppressWarnings("unchecked") public void configure() { /** diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java index 177a0a9d..099caf2b 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SendCallbackRoute.java @@ -27,6 +27,7 @@ public class SendCallbackRoute extends BaseRouteBuilder { private String callbackUrl; @Override + @SuppressWarnings("unchecked") public void configure() throws Exception { from("rest:get:test/send/callback").to(RouteId.SEND_CALLBACK.getValue()); @@ -85,6 +86,7 @@ public void configure() throws Exception { }); } + @SuppressWarnings("unchecked") public void eliminatePhases(Exchange exchange) { List phases = exchange.getProperty(PHASES, List.class); int completionRate = exchange.getProperty(COMPLETION_RATE, Integer.class); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java index 269d59b7..4cffdc80 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/SplittingRoute.java @@ -52,6 +52,7 @@ public class SplittingRoute extends BaseRouteBuilder { private boolean isPartyLookupEnabled; @Override + @SuppressWarnings("unchecked") public void configure() throws Exception { /** @@ -183,8 +184,10 @@ public void configure() throws Exception { // generate subBatchEntityDetails, make sure [LOCAL_FILE_PATH] has the absolute sub batch file path from("direct:generate-sub-batch-entity").id("direct:generate-sub-batch-entity").log("Generating sub batch entity") .to("direct:get-transaction-array").process(exchange -> { - List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); - Map zeebeVariables = exchange.getProperty(ZEEBE_VARIABLE, Map.class); + @SuppressWarnings("unchecked") + List transactionList = (List) exchange.getProperty(TRANSACTION_LIST, List.class); + @SuppressWarnings("unchecked") + Map zeebeVariables = (Map) exchange.getProperty(ZEEBE_VARIABLE, Map.class); String serverFileName = exchange.getProperty(LOCAL_FILE_PATH, String.class); logger.info("Generating sub batch entity for file {}", serverFileName); diff --git a/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java index 8efc7ee2..03b1e2a5 100644 --- a/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java +++ b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java @@ -60,30 +60,39 @@ @Service public class ProcessorStartRouteService { + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + @Autowired - ProcessorStartRoute processorStartRoute; + private ProcessorStartRoute processorStartRoute; + @Autowired @Qualifier("awsStorage") private FileTransferService fileTransferService; + @Autowired private ZeebeProcessStarter zeebeProcessStarter; + @Autowired - PhaseUtils phaseUtils; + private PhaseUtils phaseUtils; + @Autowired - public ObjectMapper objectMapper; + private ObjectMapper objectMapper; + @Autowired - BudgetAccountConfig budgetAccountConfig; + private BudgetAccountConfig budgetAccountConfig; + @Value("${application.bucket-name}") private String bucketName; + @Value("${csv.size}") private int csvSize; + @Value("${pollingApi.path}") private String pollApiPath; + @Value("${pollingApi.timer}") private String pollApiTimer; - public Logger logger = LoggerFactory.getLogger(this.getClass()); - public void validateFileSyncResponse(Exchange exchange) throws IOException { String fileName = exchange.getIn().getHeader(FILE_NAME, String.class); File file = new File(fileName); @@ -92,10 +101,10 @@ public void validateFileSyncResponse(Exchange exchange) throws IOException { int fileSize = (int) file.length(); if (fileSize > csvSize) { processorStartRoute.setErrorResponse(exchange, 400, "File too big", - "The file uploaded is too big. " + "Please upload a file and try again."); + "The file uploaded is too big. Please upload a file and try again."); } else if (!processorStartRoute.verifyCsv(file)) { processorStartRoute.setErrorResponse(exchange, 400, "Invalid file structure", - "The file uploaded contains wrong structure." + " Please upload correct file columns and try again."); + "The file uploaded contains wrong structure. Please upload correct file columns and try again."); } else { logger.debug("Filename: {}", fileName); processorStartRoute.setResponse(exchange, 200); @@ -106,8 +115,9 @@ public void validateTenant(Exchange exchange) { String tenantName = exchange.getIn().getHeader(HEADER_PLATFORM_TENANT_ID, String.class); // validation is disabled for now /* - * if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { throw new - * Exception("Invalid tenant value."); } + * if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { + * throw new Exception("Invalid tenant value."); + * } */ exchange.setProperty(TENANT_NAME, tenantName); exchange.getIn().setHeader(CONTENT_TYPE, "application/json;charset=UTF-8"); @@ -150,27 +160,33 @@ public void startBatchProcessRaw(Exchange exchange) { exchange.getIn().setBody(response.toString()); } + @SuppressWarnings("unchecked") public void updateIncomingData(Exchange exchange) { String registeringInstituteId = exchange.getProperty(REGISTERING_INSTITUTE_ID, String.class); String programId = exchange.getProperty(PROGRAM_ID, String.class); logger.debug("Inst id: {}, prog id: {}", registeringInstituteId, programId); + if (!(StringUtils.hasText(registeringInstituteId) && StringUtils.hasText(programId))) { // this will make sure the file is not updated since there is no update in data logger.debug("InstitutionId or programId is null"); - exchange.setProperty(IS_UPDATED, false); return; } + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); logger.debug("Size: {}", transactionList.size()); - RegisteringInstitutionConfig registeringInstitutionConfig = budgetAccountConfig.getByRegisteringInstituteId(registeringInstituteId); + + RegisteringInstitutionConfig registeringInstitutionConfig = + budgetAccountConfig.getByRegisteringInstituteId(registeringInstituteId); + if (registeringInstitutionConfig == null) { - logger.debug("Element in nested in config: {}", budgetAccountConfig.getRegisteringInstitutions().get(0).getPrograms().size()); + logger.debug("Element in nested in config: {}", + budgetAccountConfig.getRegisteringInstitutions().get(0).getPrograms().size()); logger.debug("Registering institute id is null"); - exchange.setProperty(IS_UPDATED, false); return; } + Program program = registeringInstitutionConfig.getByProgramId(programId); if (program == null) { // this will make sure the file is not updated since there is no update in data @@ -178,6 +194,7 @@ public void updateIncomingData(Exchange exchange) { exchange.setProperty(IS_UPDATED, false); return; } + List resultTransactionList = new ArrayList<>(); transactionList.forEach(transaction -> { @@ -190,6 +207,7 @@ public void updateIncomingData(Exchange exchange) { throw new RuntimeException(e); } }); + exchange.setProperty(RESULT_TRANSACTION_LIST, resultTransactionList); exchange.setProperty(IS_UPDATED, true); exchange.setProperty(PROGRAM_NAME, program.getName()); @@ -234,6 +252,7 @@ public void startBatchProcessCsv(Exchange exchange) throws IOException { List phases = phaseUtils.getValues(); logger.debug(phases.toString()); + Map variables = new HashMap<>(); variables.put(BATCH_ID, batchId); variables.put(FILE_NAME, fileName); @@ -257,19 +276,23 @@ public void startBatchProcessCsv(Exchange exchange) throws IOException { logger.info("Variables published to zeebe: {}", variables); JSONObject response = new JSONObject(); - String bpmn = processorStartRoute.getWorkflowForTenant(exchange.getProperty(TENANT_NAME).toString(), "batch-transactions"); + String bpmn = processorStartRoute.getWorkflowForTenant( + exchange.getProperty(TENANT_NAME).toString(), "batch-transactions"); try { logger.info("FREDa "); logger.info("FREDa: tenant is < {} > ", exchange.getProperty(TENANT_NAME).toString()); - String tenantSpecificWorkflowId = bpmn.replace("{dfspid}", exchange.getProperty(TENANT_NAME).toString()); + String tenantSpecificWorkflowId = bpmn.replace("{dfspid}", + exchange.getProperty(TENANT_NAME).toString()); logger.info("Tenant specific workflow id: {}", tenantSpecificWorkflowId); logger.info("FRED: tenant is < {} > ", exchange.getProperty(TENANT_NAME).toString()); + String txnId = zeebeProcessStarter.startZeebeWorkflow(tenantSpecificWorkflowId, "", variables); if (txnId == null || txnId.isEmpty()) { response.put("errorCode", 500); response.put("errorDescription", "Unable to start zeebe workflow"); - response.put("developerMessage", "Issue in starting the zeebe workflow, check the zeebe configuration"); + response.put("developerMessage", + "Issue in starting the zeebe workflow, check the zeebe configuration"); } else { response.put("batch_id", batchId); response.put("request_id", requestId); @@ -286,4 +309,4 @@ public void startBatchProcessCsv(Exchange exchange) throws IOException { exchange.getIn().setBody(response.toString()); } -} +} \ No newline at end of file diff --git a/src/main/java/org/mifos/processor/bulk/service/BatchAccountLookup.java b/src/main/java/org/mifos/processor/bulk/service/BatchAccountLookup.java index 77ea2f99..ebd98be8 100644 --- a/src/main/java/org/mifos/processor/bulk/service/BatchAccountLookup.java +++ b/src/main/java/org/mifos/processor/bulk/service/BatchAccountLookup.java @@ -9,7 +9,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; +//import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.camel.Exchange; @@ -21,32 +21,44 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +@Component public class BatchAccountLookup { + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + @Autowired private ObjectMapper objectMapper; + @Autowired private AccountLookupService accountLookupService; + @Value("${identity_account_mapper.hostname}") private String identityEndpoint; + @Value("${identity_account_mapper.batch_account_lookup}") private String batchAccountLookup; - public Logger logger = LoggerFactory.getLogger(this.getClass()); - + @SuppressWarnings("unchecked") public void doBatchAccountLookup(Exchange exchange) throws IOException { List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); - HashMap> stringListHashMap = new HashMap<>(); List beneficiaryDTOList = new ArrayList<>(); + transactionList.forEach(transaction -> { beneficiaryDTOList.add(new BeneficiaryDTO(transaction.getPayeeIdentifier(), "", "", "")); }); + String requestId = exchange.getProperty(REQUEST_ID, String.class); String callbackUrl = exchange.getProperty(CALLBACK, String.class); String registeringInstitutionId = exchange.getProperty(HEADER_REGISTERING_INSTITUTE_ID, String.class); - AccountMapperRequestDTO accountMapperRequestDTO = new AccountMapperRequestDTO(requestId, registeringInstitutionId, - beneficiaryDTOList); + + AccountMapperRequestDTO accountMapperRequestDTO = new AccountMapperRequestDTO( + requestId, + registeringInstitutionId, + beneficiaryDTOList + ); + String requestBody = objectMapper.writeValueAsString(accountMapperRequestDTO); exchange.getIn().setHeader(CALLBACK, callbackUrl); @@ -54,10 +66,9 @@ public void doBatchAccountLookup(Exchange exchange) throws IOException { exchange.getIn().setHeader("Content-type", "application/json"); exchange.getIn().setBody(requestBody); - Map headers = new HashMap<>(); - headers = exchange.getIn().getHeaders(); - + Map headers = exchange.getIn().getHeaders(); String fullUrl = identityEndpoint + batchAccountLookup; + accountLookupService.accountLookupCall(identityEndpoint, fullUrl, accountMapperRequestDTO, headers); } -} +} \ No newline at end of file diff --git a/src/main/java/org/mifos/processor/bulk/service/FileProcessingRouteService.java b/src/main/java/org/mifos/processor/bulk/service/FileProcessingRouteService.java index 4a803920..c09d8619 100644 --- a/src/main/java/org/mifos/processor/bulk/service/FileProcessingRouteService.java +++ b/src/main/java/org/mifos/processor/bulk/service/FileProcessingRouteService.java @@ -59,6 +59,7 @@ public void getTxnArray(Exchange exchange) throws IOException { exchange.setProperty(COMPLETED_AMOUNT, completedAmount); } + @SuppressWarnings("unchecked") public void updateResultFile(Exchange exchange) throws IOException { String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); List transactionList = exchange.getProperty(RESULT_TRANSACTION_LIST, List.class); @@ -68,6 +69,7 @@ public void updateResultFile(Exchange exchange) throws IOException { CsvWriter.writeToCsv(transactionList, TransactionResult.class, csvMapper, overrideHeader, filepath); } + @SuppressWarnings("unchecked") public void updateFile(Exchange exchange) throws IOException { String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java index b5aa4159..e35c2382 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java @@ -34,7 +34,6 @@ private static List toStringList(Object obj) { if (item != null) { result.add(item.toString()); } - logger.debug("BatchEntity for this subbatch is {}", objectMapper.writeValueAsString(subBatchEntity)); } return result; } From 73b724b01dbbac08370243867e9a3689d2447838 Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Fri, 28 Nov 2025 23:29:28 +0000 Subject: [PATCH 142/156] solving cors issues and debugging wip --- .../HttpClientConfigurerTrustAllCACerts.java | 2 +- .../bulk/camel/routes/InitSubBatchRoute.java | 4 +- .../bulk/camel/routes/MergeBackRoute.java | 69 +++++++------------ .../processor/bulk/config/CorsProperties.java | 3 +- .../bulk/config/PreflightCorsFilter.java | 17 ++--- .../service/ProcessorStartRouteService.java | 40 +++++------ .../bulk/service/BatchAccountLookup.java | 10 +-- .../bulk/zeebe/worker/InitSubBatchWorker.java | 24 ++----- src/main/resources/application.yaml | 33 ++++----- 9 files changed, 74 insertions(+), 128 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java b/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java index c2064227..bdc5464f 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java +++ b/src/main/java/org/mifos/processor/bulk/camel/config/HttpClientConfigurerTrustAllCACerts.java @@ -12,8 +12,8 @@ import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.PlainConnectionSocketFactory; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.TrustStrategy; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java index 7c8be8c8..3d4f9b95 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/InitSubBatchRoute.java @@ -128,7 +128,9 @@ public void configure() throws Exception { variables.put(DEBULKINGDFSPID, mapping.getDebulkingDfspid() == null ? tenantName : mapping.getDebulkingDfspid()); if (isPartyLookupEnabled && !(Boolean) variables.get(PARTY_LOOKUP_FAILED)) { String filename = exchange.getProperty(SERVER_FILE_NAME).toString(); - String regex = ".*_sub-batch-([\\w-]+)\\.csv"; //payee DFSP Id for sub batch are extracted from the sub batch file name when party lookup is enabled and it is successful + String regex = ".*_sub-batch-([\\w-]+)\\.csv"; // payee DFSP Id for sub batch are extracted from + // the sub batch file name when party lookup is + // enabled and it is successful Pattern pattern = Pattern.compile(regex); Matcher matcher = pattern.matcher(filename); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java index 44cb4f50..63969efa 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/MergeBackRoute.java @@ -24,38 +24,24 @@ public class MergeBackRoute extends BaseRouteBuilder { public void configure() throws Exception { /** - * Base route for kicking off merge back logic. Performs below tasks: - * 1. Picks the top two files from the array of files to be merged. - * 2. Merges them into single CSV. - * 3. Uploads the CSV to S3. - * 4. Updates the exchange variables. + * Base route for kicking off merge back logic. Performs below tasks: 1. Picks the top two files from the array + * of files to be merged. 2. Merges them into single CSV. 3. Uploads the CSV to S3. 4. Updates the exchange + * variables. */ - from(RouteId.MERGE_BACK.getValue()) - .id(RouteId.MERGE_BACK.getValue()) - .log("Starting route " + RouteId.MERGE_BACK.name()) - .choice() - .when(exchange -> exchange.getProperty(MERGE_FILE_LIST, List.class).size() == 0) - .log("Nothing to merge") - .setProperty(MERGE_FAILED, constant(false)) - .setProperty(MERGE_COMPLETED, constant(true)) - .when(exchange -> exchange.getProperty(MERGE_FILE_LIST, List.class).size() == 1) - .process(exchange -> { - exchange.setProperty(MERGE_FAILED, false); - exchange.setProperty(MERGE_COMPLETED, true); - @SuppressWarnings("unchecked") - List mergeFileList = (List) exchange.getProperty(MERGE_FILE_LIST, List.class); - String resultFile = mergeFileList.get(0); - setResultFileProperty(exchange, resultFile); - }) - .otherwise() - .to("direct:start-merge") - .endChoice(); + from(RouteId.MERGE_BACK.getValue()).id(RouteId.MERGE_BACK.getValue()).log("Starting route " + RouteId.MERGE_BACK.name()).choice() + .when(exchange -> exchange.getProperty(MERGE_FILE_LIST, List.class).size() == 0).log("Nothing to merge") + .setProperty(MERGE_FAILED, constant(false)).setProperty(MERGE_COMPLETED, constant(true)) + .when(exchange -> exchange.getProperty(MERGE_FILE_LIST, List.class).size() == 1).process(exchange -> { + exchange.setProperty(MERGE_FAILED, false); + exchange.setProperty(MERGE_COMPLETED, true); + @SuppressWarnings("unchecked") + List mergeFileList = (List) exchange.getProperty(MERGE_FILE_LIST, List.class); + String resultFile = mergeFileList.get(0); + setResultFileProperty(exchange, resultFile); + }).otherwise().to("direct:start-merge").endChoice(); // starts the merge process, merges the file and uploads the file in s3 - from("direct:start-merge") - .id("direct:start-merge") - .log("Starting route direct:start-merge") - .to("direct:download-file-to-be-merged") + from("direct:start-merge").id("direct:start-merge").log("Starting route direct:start-merge").to("direct:download-file-to-be-merged") .process(exchange -> { String file1 = exchange.getProperty(FILE_1, String.class); String file2 = exchange.getProperty(FILE_2, String.class); @@ -67,16 +53,13 @@ public void configure() throws Exception { } if (exchange.getProperty(MERGE_ITERATION, Integer.class) == 1) { // generate new name for merged file in case of first iteration - String newFileName = System.currentTimeMillis() + "_" - + exchange.getProperty(BATCH_ID, String.class) + ".csv"; + String newFileName = System.currentTimeMillis() + "_" + exchange.getProperty(BATCH_ID, String.class) + ".csv"; new File(mergedFile).renameTo(new File(newFileName)); exchange.setProperty(LOCAL_FILE_PATH, newFileName); } else { exchange.setProperty(LOCAL_FILE_PATH, mergedFile); } - }) - .to("direct:upload-file") - .process(exchange -> { + }).to("direct:upload-file").process(exchange -> { String mergedFileServerName = exchange.getProperty(SERVER_FILE_NAME, String.class); @SuppressWarnings("unchecked") List mergeList = (List) exchange.getProperty(MERGE_FILE_LIST, List.class); @@ -102,23 +85,17 @@ public void configure() throws Exception { }); // downloads the two files (using FIFO access pattern) from s3 which are to be merged - from("direct:download-file-to-be-merged") - .id("direct:download-file-to-be-merged") - .log("Starting route direct:download-file-to-be-merged") - .log("Downloading files to be merged") - .process(exchange -> { + from("direct:download-file-to-be-merged").id("direct:download-file-to-be-merged") + .log("Starting route direct:download-file-to-be-merged").log("Downloading files to be merged").process(exchange -> { @SuppressWarnings("unchecked") List mergeList = (List) exchange.getProperty(MERGE_FILE_LIST, List.class); exchange.setProperty(SERVER_FILE_NAME, mergeList.get(0)); - }) - .to("direct:download-file") // downloading first file - .setProperty(FILE_1, exchangeProperty(LOCAL_FILE_PATH)) - .process(exchange -> { + }).to("direct:download-file") // downloading first file + .setProperty(FILE_1, exchangeProperty(LOCAL_FILE_PATH)).process(exchange -> { @SuppressWarnings("unchecked") List mergeList = (List) exchange.getProperty(MERGE_FILE_LIST, List.class); exchange.setProperty(SERVER_FILE_NAME, mergeList.get(1)); - }) - .to("direct:download-file") // downloading second file + }).to("direct:download-file") // downloading second file .setProperty(FILE_2, exchangeProperty(LOCAL_FILE_PATH)); } @@ -126,4 +103,4 @@ public void configure() throws Exception { public void setResultFileProperty(Exchange exchange, String fileName) { exchange.setProperty(RESULT_FILE, Utils.getAwsFileUrl(awsS3BaseUrl, fileName)); } -} \ No newline at end of file +} diff --git a/src/main/java/org/mifos/processor/bulk/config/CorsProperties.java b/src/main/java/org/mifos/processor/bulk/config/CorsProperties.java index 1741bff8..ba713a79 100644 --- a/src/main/java/org/mifos/processor/bulk/config/CorsProperties.java +++ b/src/main/java/org/mifos/processor/bulk/config/CorsProperties.java @@ -1,10 +1,9 @@ package org.mifos.processor.bulk.config; +import java.util.List; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; -import java.util.List; - @Configuration @ConfigurationProperties(prefix = "cors") public class CorsProperties { diff --git a/src/main/java/org/mifos/processor/bulk/config/PreflightCorsFilter.java b/src/main/java/org/mifos/processor/bulk/config/PreflightCorsFilter.java index f7b21354..6931055b 100644 --- a/src/main/java/org/mifos/processor/bulk/config/PreflightCorsFilter.java +++ b/src/main/java/org/mifos/processor/bulk/config/PreflightCorsFilter.java @@ -1,16 +1,15 @@ package org.mifos.processor.bulk.config; -import org.springframework.stereotype.Component; -import org.springframework.beans.factory.annotation.Autowired; - +import java.io.IOException; +import java.util.List; import javax.servlet.Filter; import javax.servlet.FilterChain; import javax.servlet.FilterConfig; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.util.List; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; @Component public class PreflightCorsFilter implements Filter { @@ -23,9 +22,8 @@ public PreflightCorsFilter(CorsProperties corsProperties) { } @Override - public void doFilter(javax.servlet.ServletRequest servletRequest, - javax.servlet.ServletResponse servletResponse, - FilterChain filterChain) throws IOException, ServletException { + public void doFilter(javax.servlet.ServletRequest servletRequest, javax.servlet.ServletResponse servletResponse, + FilterChain filterChain) throws IOException, ServletException { HttpServletRequest request = (HttpServletRequest) servletRequest; HttpServletResponse response = (HttpServletResponse) servletResponse; @@ -52,7 +50,6 @@ public void init(FilterConfig filterConfig) throws ServletException { } @Override - public void destroy() { - } + public void destroy() {} } diff --git a/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java index 03b1e2a5..dc02e636 100644 --- a/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java +++ b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java @@ -115,9 +115,8 @@ public void validateTenant(Exchange exchange) { String tenantName = exchange.getIn().getHeader(HEADER_PLATFORM_TENANT_ID, String.class); // validation is disabled for now /* - * if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { - * throw new Exception("Invalid tenant value."); - * } + * if (tenantName == null || tenantName.isEmpty() || !tenants.contains(tenantName)) { throw new + * Exception("Invalid tenant value."); } */ exchange.setProperty(TENANT_NAME, tenantName); exchange.getIn().setHeader(CONTENT_TYPE, "application/json;charset=UTF-8"); @@ -165,28 +164,26 @@ public void updateIncomingData(Exchange exchange) { String registeringInstituteId = exchange.getProperty(REGISTERING_INSTITUTE_ID, String.class); String programId = exchange.getProperty(PROGRAM_ID, String.class); logger.debug("Inst id: {}, prog id: {}", registeringInstituteId, programId); - + if (!(StringUtils.hasText(registeringInstituteId) && StringUtils.hasText(programId))) { // this will make sure the file is not updated since there is no update in data logger.debug("InstitutionId or programId is null"); exchange.setProperty(IS_UPDATED, false); return; } - + List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); logger.debug("Size: {}", transactionList.size()); - - RegisteringInstitutionConfig registeringInstitutionConfig = - budgetAccountConfig.getByRegisteringInstituteId(registeringInstituteId); - + + RegisteringInstitutionConfig registeringInstitutionConfig = budgetAccountConfig.getByRegisteringInstituteId(registeringInstituteId); + if (registeringInstitutionConfig == null) { - logger.debug("Element in nested in config: {}", - budgetAccountConfig.getRegisteringInstitutions().get(0).getPrograms().size()); + logger.debug("Element in nested in config: {}", budgetAccountConfig.getRegisteringInstitutions().get(0).getPrograms().size()); logger.debug("Registering institute id is null"); exchange.setProperty(IS_UPDATED, false); return; } - + Program program = registeringInstitutionConfig.getByProgramId(programId); if (program == null) { // this will make sure the file is not updated since there is no update in data @@ -194,7 +191,7 @@ public void updateIncomingData(Exchange exchange) { exchange.setProperty(IS_UPDATED, false); return; } - + List resultTransactionList = new ArrayList<>(); transactionList.forEach(transaction -> { @@ -207,7 +204,7 @@ public void updateIncomingData(Exchange exchange) { throw new RuntimeException(e); } }); - + exchange.setProperty(RESULT_TRANSACTION_LIST, resultTransactionList); exchange.setProperty(IS_UPDATED, true); exchange.setProperty(PROGRAM_NAME, program.getName()); @@ -252,7 +249,7 @@ public void startBatchProcessCsv(Exchange exchange) throws IOException { List phases = phaseUtils.getValues(); logger.debug(phases.toString()); - + Map variables = new HashMap<>(); variables.put(BATCH_ID, batchId); variables.put(FILE_NAME, fileName); @@ -276,23 +273,20 @@ public void startBatchProcessCsv(Exchange exchange) throws IOException { logger.info("Variables published to zeebe: {}", variables); JSONObject response = new JSONObject(); - String bpmn = processorStartRoute.getWorkflowForTenant( - exchange.getProperty(TENANT_NAME).toString(), "batch-transactions"); + String bpmn = processorStartRoute.getWorkflowForTenant(exchange.getProperty(TENANT_NAME).toString(), "batch-transactions"); try { logger.info("FREDa "); logger.info("FREDa: tenant is < {} > ", exchange.getProperty(TENANT_NAME).toString()); - String tenantSpecificWorkflowId = bpmn.replace("{dfspid}", - exchange.getProperty(TENANT_NAME).toString()); + String tenantSpecificWorkflowId = bpmn.replace("{dfspid}", exchange.getProperty(TENANT_NAME).toString()); logger.info("Tenant specific workflow id: {}", tenantSpecificWorkflowId); logger.info("FRED: tenant is < {} > ", exchange.getProperty(TENANT_NAME).toString()); - + String txnId = zeebeProcessStarter.startZeebeWorkflow(tenantSpecificWorkflowId, "", variables); if (txnId == null || txnId.isEmpty()) { response.put("errorCode", 500); response.put("errorDescription", "Unable to start zeebe workflow"); - response.put("developerMessage", - "Issue in starting the zeebe workflow, check the zeebe configuration"); + response.put("developerMessage", "Issue in starting the zeebe workflow, check the zeebe configuration"); } else { response.put("batch_id", batchId); response.put("request_id", requestId); @@ -309,4 +303,4 @@ public void startBatchProcessCsv(Exchange exchange) throws IOException { exchange.getIn().setBody(response.toString()); } -} \ No newline at end of file +} diff --git a/src/main/java/org/mifos/processor/bulk/service/BatchAccountLookup.java b/src/main/java/org/mifos/processor/bulk/service/BatchAccountLookup.java index ebd98be8..f90a42a9 100644 --- a/src/main/java/org/mifos/processor/bulk/service/BatchAccountLookup.java +++ b/src/main/java/org/mifos/processor/bulk/service/BatchAccountLookup.java @@ -9,7 +9,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.ArrayList; -//import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.camel.Exchange; @@ -53,11 +52,8 @@ public void doBatchAccountLookup(Exchange exchange) throws IOException { String callbackUrl = exchange.getProperty(CALLBACK, String.class); String registeringInstitutionId = exchange.getProperty(HEADER_REGISTERING_INSTITUTE_ID, String.class); - AccountMapperRequestDTO accountMapperRequestDTO = new AccountMapperRequestDTO( - requestId, - registeringInstitutionId, - beneficiaryDTOList - ); + AccountMapperRequestDTO accountMapperRequestDTO = new AccountMapperRequestDTO(requestId, registeringInstitutionId, + beneficiaryDTOList); String requestBody = objectMapper.writeValueAsString(accountMapperRequestDTO); @@ -71,4 +67,4 @@ public void doBatchAccountLookup(Exchange exchange) throws IOException { accountLookupService.accountLookupCall(identityEndpoint, fullUrl, accountMapperRequestDTO, headers); } -} \ No newline at end of file +} diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java index e35c2382..b86e9f7c 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java @@ -6,10 +6,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Objects; import org.apache.camel.Exchange; import org.apache.camel.support.DefaultExchange; import org.mifos.processor.bulk.camel.routes.RouteId; @@ -71,10 +69,7 @@ public void setup() { variables.put(SUB_BATCHES, new ArrayList()); variables.put(INIT_SUCCESS_SUB_BATCHES, new ArrayList()); variables.put(INIT_FAILURE_SUB_BATCHES, new ArrayList()); - client.newCompleteCommand(job.getKey()) - .variables(variables) - .send() - .join(); + client.newCompleteCommand(job.getKey()).variables(variables).send().join(); return; } @@ -91,15 +86,11 @@ public void setup() { String currentFile = subBatches.remove(0); // Parse sub-batch details - List subBatchEntityList = objectMapper.convertValue( - subBatchDetails, - new TypeReference>() {} - ); + List subBatchEntityList = objectMapper.convertValue(subBatchDetails, + new TypeReference>() {}); SubBatchEntity subBatchEntity = subBatchEntityList.stream() - .filter(e -> e.getRequestFile() != null && e.getRequestFile().contains(currentFile)) - .findFirst() - .orElse(null); + .filter(e -> e.getRequestFile() != null && e.getRequestFile().contains(currentFile)).findFirst().orElse(null); // Setup Camel exchange Exchange exchange = new DefaultExchange(camelContext); @@ -126,12 +117,9 @@ public void setup() { variables.put(INIT_SUCCESS_SUB_BATCHES, new ArrayList<>(successSubBatches)); variables.put(INIT_FAILURE_SUB_BATCHES, new ArrayList<>(failureSubBatches)); - client.newCompleteCommand(job.getKey()) - .variables(variables) - .send() - .join(); + client.newCompleteCommand(job.getKey()).variables(variables).send().join(); logger.info("Completed INIT_SUB_BATCH worker. Remaining sub-batches: {}", subBatches.size()); }); } -} \ No newline at end of file +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index fe5b9953..54f5232b 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -1,16 +1,11 @@ -# camel: -# server-port: 5000 -# disable-ssl: false -# springboot: -# main-run-controller: true -# dataformat: -# json-jackson: -# auto-discover-object-mapper: true - -# TD Nov 2025 -- terminate TLS at NGINX run pod on http => solve cors issues from operations-web camel: server-port: 0 - disable-ssl: true + disable-ssl: false + springboot: + main-run-controller: true + dataformat: + json-jackson: + auto-discover-object-mapper: true cors: allowed-origins: @@ -130,13 +125,13 @@ callback-phases: - 100 server: - # ssl: - # key-alias: "tomcat-https" - # key-store: "classpath:keystore.jks" - # key-store-type: JKS - # key-password: "password" - # key-store-password: "password" - port: 8080 + ssl: + key-alias: "tomcat-https" + key-store: "classpath:keystore.jks" + key-store-type: JKS + key-password: "password" + key-store-password: "password" + port: 8443 # HTTPS port for Spring Boot (external traffic, gRPC) security: @@ -199,8 +194,6 @@ payment-mode: batch-authorization: callback-url: "${BULK_PROCESSOR_CALLBACK_URL:https://bulk-processor.mifos.gazelle.test/authorization/callback}" - - pubsub: room: code: "covid-19" From 37baa74c3fa01743d27d58e66f5ee6e79582cff4 Mon Sep 17 00:00:00 2001 From: Devarsh Date: Wed, 17 Dec 2025 15:33:07 +0530 Subject: [PATCH 143/156] change base image to eclipse-temurin:17-jdk --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4bad37e3..d93b06ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM openjdk:17 +FROM eclipse-temurin:17-jdk EXPOSE 5000 COPY build/libs/*.jar . From 86136f8e1efa5e7f6525b303850500199c7375b2 Mon Sep 17 00:00:00 2001 From: devarsh10 Date: Wed, 17 Dec 2025 15:33:37 +0530 Subject: [PATCH 144/156] Update CircleCI config for Java 17 --- .circleci/config.yml | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 930d5efa..16f0e510 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,7 @@ orbs: executors: docker-executor: docker: - - image: circleci/openjdk:17-buster-node-browsers-legacy + - image: eclipse-temurin:17-jdk jobs: build_and_push_tag_image: executor: docker-executor @@ -23,6 +23,18 @@ jobs: echo "export DOCKER_REPO_LOWER=$(echo $CIRCLE_PROJECT_REPONAME | tr '[:upper:]' '[:lower:]')" >> $BASH_ENV echo "Using Docker Namespace: $DOCKER_ORG_LOWER" echo "Using Docker Repository: $DOCKER_REPO_LOWER" + - run: + name: Install Docker CLI + command: | + apt-get update + apt-get install -y curl ca-certificates gnupg + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian bullseye stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null + apt-get update + apt-cache madison docker-ce-cli | grep 20.10 + apt-get install -y docker-ce-cli=5:20.10.24~3-0~debian-bullseye - docker-buildx/install - run: name: Check if Docker image tag exists @@ -64,6 +76,18 @@ jobs: echo "export DOCKER_REPO_LOWER=$(echo $CIRCLE_PROJECT_REPONAME | tr '[:upper:]' '[:lower:]')" >> $BASH_ENV echo "Using Docker Namespace: $DOCKER_ORG_LOWER" echo "Using Docker Repository: $DOCKER_REPO_LOWER" + - run: + name: Install Docker CLI + command: | + apt-get update + apt-get install -y curl ca-certificates gnupg + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian bullseye stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null + apt-get update + apt-cache madison docker-ce-cli | grep 20.10 + apt-get install -y docker-ce-cli=5:20.10.24~3-0~debian-bullseye - docker-buildx/install - run: name: Build Application @@ -101,6 +125,18 @@ jobs: echo "export DOCKER_REPO_LOWER=$(echo $CIRCLE_PROJECT_REPONAME | tr '[:upper:]' '[:lower:]')" >> $BASH_ENV echo "Using Docker Namespace: $DOCKER_ORG_LOWER" echo "Using Docker Repository: $DOCKER_REPO_LOWER" + - run: + name: Install Docker CLI + command: | + apt-get update + apt-get install -y curl ca-certificates gnupg + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian bullseye stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null + apt-get update + apt-cache madison docker-ce-cli | grep 20.10 + apt-get install -y docker-ce-cli=5:20.10.24~3-0~debian-bullseye - docker-buildx/install - run: name: Build Application @@ -147,4 +183,4 @@ workflows: only: - main # Or your primary branch name like 'master' context: - - DOCKER + - DOCKER \ No newline at end of file From 5e449358b23fd9ce06e43fe0235839a724a28104 Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Sat, 20 Dec 2025 09:30:56 +0000 Subject: [PATCH 145/156] a bunch of changes . most for non-govstack but some initial govstack mods/bug fixes --- .../api/definition/BatchTransactions.java | 9 +++ .../BatchTransactionsController.java | 78 ++++++++++++++++++- .../camel/routes/BatchAggregateRoute.java | 1 + .../camel/routes/FileProcessingRoute.java | 12 ++- .../camel/routes/ProcessorStartRoute.java | 42 +++++----- .../bulk/file/AwsFileTransferImpl.java | 8 ++ .../bulk/file/FileStorageServiceImpl.java | 13 +++- .../processor/bulk/schema/Transaction.java | 2 +- .../service/FileProcessingRouteService.java | 12 ++- .../mifos/processor/bulk/utility/Headers.java | 5 +- .../worker/BatchAccountLookupWorker.java | 3 +- .../bulk/zeebe/worker/InitSubBatchWorker.java | 18 ++++- src/main/resources/application.yaml | 25 +++--- 13 files changed, 183 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/api/definition/BatchTransactions.java b/src/main/java/org/mifos/processor/bulk/api/definition/BatchTransactions.java index 34386c38..e7fb0a02 100644 --- a/src/main/java/org/mifos/processor/bulk/api/definition/BatchTransactions.java +++ b/src/main/java/org/mifos/processor/bulk/api/definition/BatchTransactions.java @@ -13,8 +13,11 @@ import java.io.IOException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.springframework.http.MediaType; +import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestParam; public interface BatchTransactions { @@ -28,4 +31,10 @@ String batchTransactions(HttpServletRequest httpServletRequest, HttpServletRespo @RequestHeader(value = CALLBACK, required = false) String callbackUrl, @RequestHeader(value = PAYEE_DFSP_ID, required = false) String payeeDfspId) throws IOException; + // NEW execution reconciliation endpoint + @PostMapping(value = "/batchtransactions/execution", consumes = MediaType.APPLICATION_FORM_URLENCODED_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + String updateBatchExecution(@RequestParam MultiValueMap executionPayload, + @RequestHeader(value = HEADER_PLATFORM_TENANT_ID) String tenant, + @RequestHeader(value = HEADER_CLIENT_CORRELATION_ID, required = false) String requestId); + } diff --git a/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java b/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java index 5469ff32..2d2ead74 100644 --- a/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java +++ b/src/main/java/org/mifos/processor/bulk/api/implementation/BatchTransactionsController.java @@ -39,6 +39,7 @@ import org.mifos.processor.bulk.utility.SpringWrapperUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartException; @@ -84,7 +85,24 @@ public String batchTransactions(HttpServletRequest httpServletRequest, HttpServl if (JWSUtil.isMultipartRequest(httpServletRequest)) { log.info("This is file based request"); - String localFileName = fileStorageService.save(JWSUtil.parseFormData(httpServletRequest), fileName); + // Use Spring's MultipartHttpServletRequest instead of JWSUtil.parseFormData() + org.springframework.web.multipart.MultipartHttpServletRequest multipartRequest = (org.springframework.web.multipart.MultipartHttpServletRequest) httpServletRequest; + org.springframework.web.multipart.MultipartFile multipartFile = multipartRequest.getFile("data"); + + log.info("multipartFile is null: {}", multipartFile == null); + if (multipartFile != null) { + log.info("multipartFile.isEmpty(): {}", multipartFile.isEmpty()); + log.info("multipartFile.getSize(): {}", multipartFile.getSize()); + log.info("multipartFile.getOriginalFilename(): {}", multipartFile.getOriginalFilename()); + } + + if (multipartFile == null || multipartFile.isEmpty()) { + log.error("No file data found in multipart request"); + httpServletResponse.setStatus(HttpServletResponse.SC_BAD_REQUEST); + return getErrorResponse("No file data", "No file was uploaded with the request", HttpServletResponse.SC_BAD_REQUEST); + } + + String localFileName = fileStorageService.save(multipartFile); Headers headers = headerBuilder.addHeader(FILE_NAME, localFileName).build(); log.info("Headers passed: {}", headers.getHeaders()); @@ -162,4 +180,62 @@ private void checkAndThrowClientStatusException(Exchange exchange) { throw new ClientStatusException(Status.FAILED_PRECONDITION, cause); } } + + @Override + public String updateBatchExecution(MultiValueMap executionPayload, String tenant, String requestId) { + + log.info("## BATCH EXECUTION UPDATE - Received execution update request"); + log.info("## Tenant: {}", tenant); + log.info("## Request ID: {}", requestId); + log.info("## Payload keys: {}", executionPayload.keySet()); + log.info("## Payload size: {}", executionPayload.size()); + + // Log all payload data for debugging + executionPayload.forEach((key, value) -> { + log.info("## Execution payload - {}: {}", key, value); + }); + + try { + // Extract transaction results from payload + // Expected format: status, transactionId, completedTimestamp, etc. + String status = getPayloadValue(executionPayload, "status"); + String transactionId = getPayloadValue(executionPayload, "transactionId"); + String batchId = getPayloadValue(executionPayload, "batchId"); + String subBatchId = getPayloadValue(executionPayload, "subBatchId"); + + log.info("## Processing execution update - batchId: {}, subBatchId: {}, transactionId: {}, status: {}", batchId, subBatchId, + transactionId, status); + + // TODO: Update transaction status in database/Zeebe workflow + // For now, just acknowledge receipt + + JSONObject response = new JSONObject(); + response.put("message", "Batch execution update received successfully"); + response.put("requestId", requestId); + response.put("batchId", batchId); + response.put("transactionId", transactionId); + response.put("status", "ACCEPTED"); + + log.info("## BATCH EXECUTION UPDATE - Successfully processed update for transactionId: {}", transactionId); + + return response.toString(); + + } catch (Exception e) { + log.error("## BATCH EXECUTION UPDATE - Error processing execution update", e); + JSONObject errorResponse = new JSONObject(); + errorResponse.put("error", "Failed to process execution update"); + errorResponse.put("message", e.getMessage()); + errorResponse.put("requestId", requestId); + return errorResponse.toString(); + } + } + + private String getPayloadValue(MultiValueMap payload, String key) { + List values = payload.get(key); + if (values != null && !values.isEmpty()) { + Object value = values.get(0); + return value != null ? value.toString() : null; + } + return null; + } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java index 265147d8..6a89e9fa 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/BatchAggregateRoute.java @@ -49,6 +49,7 @@ public void configure() throws Exception { .setHeader(HEADER_PLATFORM_TENANT_ID, simple("${exchangeProperty." + TENANT_ID + "}")).process(exchange -> { logger.info(exchange.getIn().getHeaders().toString()); }).toD(operationsAppConfig.batchAggregateUrl + "${exchangeProperty." + BATCH_ID + "}?bridgeEndpoint=true") + .log(LoggingLevel.INFO, "Actual dynamic endpoint called: ${header.CamelToEndpoint}") .log(LoggingLevel.INFO, "Batch aggregate API response: \n\n ${body}") .log(LoggingLevel.INFO, "Aggregate Response body: ${body}"); diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java index cf40f128..26605d64 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java @@ -99,6 +99,10 @@ public void configure() { String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + log.info("update-file-v2 - filepath: {}", filepath); + log.info("update-file-v2 - transactionList: {} (size: {})", transactionList != null ? "present" : "NULL", + transactionList != null ? transactionList.size() : 0); + // getting header Boolean overrideHeader = exchange.getProperty(OVERRIDE_HEADER, Boolean.class); CsvSchema csvSchema = csvMapper.schemaFor(Transaction.class); @@ -110,9 +114,13 @@ public void configure() { File file = new File(filepath); SequenceWriter writer = csvMapper.writerWithSchemaFor(Transaction.class).with(csvSchema).writeValues(file); - for (Transaction transaction : transactionList) { - writer.write(transaction); + if (transactionList != null) { + for (Transaction transaction : transactionList) { + writer.write(transaction); + } } + writer.close(); + log.info("update-file-v2 - wrote {} transactions to {}", transactionList != null ? transactionList.size() : 0, filepath); }); } } diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java index b433ca78..46ff527f 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/ProcessorStartRoute.java @@ -158,7 +158,13 @@ private void setup() { // config from("direct:update-incoming-data").id("direct:update-incoming-data").log("direct:update-incoming-data") // [LOCAL_FILE_PATH] is already set in [direct:validateFileSyncResponse] route - .setProperty(LOCAL_FILE_PATH, exchangeProperty(FILE_NAME)).to("direct:get-transaction-array") + // FIXED: Prepend "/" because files are saved to root directory by FileStorageServiceImpl + // (Paths.get("")) + .process(exchange -> { + String filename = exchange.getProperty(FILE_NAME, String.class); + String fullPath = "/" + filename; + exchange.setProperty(LOCAL_FILE_PATH, fullPath); + }).to("direct:get-transaction-array") // make sure new data is set under the exchange variable [RESULT_TRANSACTION_LIST] .bean(ProcessorStartRouteService.class, "updateIncomingData").choice() // update only when previous(edit function) makes any changes to data @@ -239,22 +245,30 @@ private void setup() { } public boolean verifyData(File file) throws IOException { + logger.info("verifyData() - file exists: {}, file size: {}, file path: {}", file.exists(), file.length(), file.getAbsolutePath()); InputStream ips = new FileInputStream(file); InputStreamReader ipsr = new InputStreamReader(ips); BufferedReader br = new BufferedReader(ipsr); String line; - br.readLine(); + String header = br.readLine(); + logger.info("verifyData() - CSV header: {}", header); + int rowCount = 0; while ((line = br.readLine()) != null) { + rowCount++; String[] row = line.split(","); + logger.info("verifyData() - row {}: length={}, expected={}, content={}", rowCount, row.length, columnNames.size(), line); if (row.length != columnNames.size()) { + logger.info("verifyData() - Row invalid: length={}, expected={}", row.length, columnNames.size()); logger.debug("DEBUG FRED10 Row invalid {} {}", row.length, columnNames.size()); logger.debug("Row invalid {} {}", row.length, columnNames.size()); return false; } if (!verifyRow(row)) { + logger.info("verifyData() - verifyRow failed for row {}", rowCount); return false; } } + logger.info("verifyData() - SUCCESS: validated {} rows", rowCount); return true; } @@ -272,25 +286,11 @@ public String getWorkflowForTenant(String tenantId, String useCase) { } private boolean verifyRow(String[] row) { - for (int i = 1; i < row.length; i++) { - row[i] = row[i].trim(); - if (row[i].equalsIgnoreCase("MSISDN")) { - int j = row[i].indexOf("MSISDN"); - if (!(j == row.length)) { - if (!row[j + 1].matches("^[0-9]*$")) { - logger.debug("MSISDN invalid"); - return false; - } - } - } else if (row[i].contains("amount")) { - int j = row[i].indexOf("amount"); - if (!row[j].matches("^[0-9]*$")) { - logger.debug("Amount invalid"); - return false; - } - - } - } + // DISABLED: This validation logic is broken - it checks row VALUES instead of using column positions + // Example bug: When it sees "MSISDN" at position 3, it does indexOf("MSISDN") which returns 0, + // then checks row[1] (the UUID request_id) instead of row[4] (the actual phone number) + // TODO: Rewrite this to use column positions from columnNames instead of searching values + logger.info("verifyRow() - VALIDATION DISABLED - row accepted"); return true; } diff --git a/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java b/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java index ec9dcd56..945729aa 100644 --- a/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java +++ b/src/main/java/org/mifos/processor/bulk/file/AwsFileTransferImpl.java @@ -27,14 +27,18 @@ public class AwsFileTransferImpl implements FileTransferService { @Override public byte[] downloadFile(String fileName, String bucketName) { + logger.info("downloadFile() - downloading from MinIO: bucket={}, file={}", bucketName, fileName); S3Object s3Object = s3Client.getObject(bucketName, fileName); S3ObjectInputStream inputStream = s3Object.getObjectContent(); try { byte[] content = IOUtils.toByteArray(inputStream); + logger.info("downloadFile() - downloaded {} bytes", content != null ? content.length : 0); return content; } catch (IOException e) { + logger.error("downloadFile() - IOException: {}", e.getMessage(), e); logger.debug("{}", e.getMessage()); } + logger.info("downloadFile() - returning null (download failed)"); return null; } @@ -48,8 +52,12 @@ public String uploadFile(MultipartFile file, String bucketName) { @Override public String uploadFile(File file, String bucketName) { String fileName = file.getName(); + logger.info("uploadFile() - file: {}, size: {}, exists: {}, path: {}", fileName, file.length(), file.exists(), + file.getAbsolutePath()); s3Client.putObject(new PutObjectRequest(bucketName, fileName, file)); + logger.info("uploadFile() - uploaded to MinIO bucket: {}", bucketName); file.delete(); + logger.info("uploadFile() - local file deleted"); return fileName; } diff --git a/src/main/java/org/mifos/processor/bulk/file/FileStorageServiceImpl.java b/src/main/java/org/mifos/processor/bulk/file/FileStorageServiceImpl.java index 2196cf21..445cbc92 100644 --- a/src/main/java/org/mifos/processor/bulk/file/FileStorageServiceImpl.java +++ b/src/main/java/org/mifos/processor/bulk/file/FileStorageServiceImpl.java @@ -7,20 +7,31 @@ import java.nio.file.Paths; import java.util.Objects; import java.util.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; @Service public class FileStorageServiceImpl implements FileStorageService { + private static final Logger logger = LoggerFactory.getLogger(FileStorageServiceImpl.class); + private final Path root = Paths.get(""); @Override public String save(MultipartFile file) { String filename = UUID.randomUUID() + "_" + Objects.requireNonNull(file.getOriginalFilename()); + logger.info("FileStorageServiceImpl.save() - filename: {}", filename); + logger.info("FileStorageServiceImpl.save() - file.getSize(): {}", file.getSize()); + logger.info("FileStorageServiceImpl.save() - file.isEmpty(): {}", file.isEmpty()); try { - Files.copy(file.getInputStream(), this.root.resolve(filename)); + InputStream inputStream = file.getInputStream(); + logger.info("FileStorageServiceImpl.save() - inputStream available: {}", inputStream.available()); + long bytesCopied = Files.copy(inputStream, this.root.resolve(filename)); + logger.info("FileStorageServiceImpl.save() - bytesCopied: {}", bytesCopied); } catch (Exception e) { + logger.error("FileStorageServiceImpl.save() - Exception: {}", e.getMessage(), e); throw new RuntimeException("Failed to save file: " + e.getMessage(), e); } return filename; diff --git a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java index 85e70800..ddaafd69 100644 --- a/src/main/java/org/mifos/processor/bulk/schema/Transaction.java +++ b/src/main/java/org/mifos/processor/bulk/schema/Transaction.java @@ -12,7 +12,7 @@ @Setter @JsonIgnoreProperties(ignoreUnknown = true) @JsonPropertyOrder({ "id", "request_id", "payment_mode", "payer_identifier_type", "payer_identifier", "payee_identifier_type", - "payee_identifier", "amount", "currency", "note", "program_shortcode", "cycle", "payee_dfsp_id", "batch_id", "account_number" }) + "payee_identifier", "amount", "currency", "note" }) public class Transaction implements CsvSchema { @JsonProperty("id") diff --git a/src/main/java/org/mifos/processor/bulk/service/FileProcessingRouteService.java b/src/main/java/org/mifos/processor/bulk/service/FileProcessingRouteService.java index c09d8619..84e11ac8 100644 --- a/src/main/java/org/mifos/processor/bulk/service/FileProcessingRouteService.java +++ b/src/main/java/org/mifos/processor/bulk/service/FileProcessingRouteService.java @@ -74,6 +74,10 @@ public void updateFile(Exchange exchange) throws IOException { String filepath = exchange.getProperty(LOCAL_FILE_PATH, String.class); List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); + log.info("updateFile() - filepath: {}", filepath); + log.info("updateFile() - transactionList: {} (size: {})", transactionList != null ? "present" : "NULL", + transactionList != null ? transactionList.size() : 0); + // getting header Boolean overrideHeader = exchange.getProperty(OVERRIDE_HEADER, Boolean.class); CsvSchema csvSchema = csvMapper.schemaFor(Transaction.class); @@ -85,8 +89,12 @@ public void updateFile(Exchange exchange) throws IOException { File file = new File(filepath); SequenceWriter writer = csvMapper.writerWithSchemaFor(Transaction.class).with(csvSchema).writeValues(file); - for (Transaction transaction : transactionList) { - writer.write(transaction); + if (transactionList != null) { + for (Transaction transaction : transactionList) { + writer.write(transaction); + } } + writer.close(); + log.info("updateFile() - wrote {} transactions to {}", transactionList != null ? transactionList.size() : 0, filepath); } } diff --git a/src/main/java/org/mifos/processor/bulk/utility/Headers.java b/src/main/java/org/mifos/processor/bulk/utility/Headers.java index f46aba91..16dcde77 100644 --- a/src/main/java/org/mifos/processor/bulk/utility/Headers.java +++ b/src/main/java/org/mifos/processor/bulk/utility/Headers.java @@ -29,7 +29,10 @@ public Object get(String key) { public static Map convertHeaders(Map headers) { Map stringHeaders = new HashMap<>(); for (Map.Entry entry : headers.entrySet()) { - stringHeaders.put(entry.getKey(), entry.getValue().toString()); + Object value = entry.getValue(); + if (value != null) { + stringHeaders.put(entry.getKey(), value.toString()); + } } return stringHeaders; } diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupWorker.java index a4861ba4..f1cc8c6d 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupWorker.java @@ -49,7 +49,8 @@ public void setup() { Map variables = job.getVariablesAsMap(); Exchange exchange = new DefaultExchange(camelContext); String filename = (String) variables.get(FILE_NAME); - String registeringInstituteId = variables.get(REGISTERING_INSTITUTE_ID).toString(); + Object registeringInstituteIdObj = variables.get(REGISTERING_INSTITUTE_ID); + String registeringInstituteId = registeringInstituteIdObj != null ? registeringInstituteIdObj.toString() : null; logger.info("registeringInstituteId in worker {}", registeringInstituteId); variables.put(CACHED_TRANSACTION_ID, job.getKey()); exchange.setProperty(HEADER_REGISTERING_INSTITUTE_ID, registeringInstituteId); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java index b86e9f7c..f4fe7f76 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/InitSubBatchWorker.java @@ -1,7 +1,21 @@ package org.mifos.processor.bulk.zeebe.worker; -import static org.mifos.processor.bulk.camel.config.CamelProperties.*; -import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SERVER_FILE_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_DETAILS; +import static org.mifos.processor.bulk.camel.config.CamelProperties.SUB_BATCH_ENTITY; +import static org.mifos.processor.bulk.camel.config.CamelProperties.TENANT_NAME; +import static org.mifos.processor.bulk.camel.config.CamelProperties.ZEEBE_VARIABLE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_FAILURE_SUB_BATCHES; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_SUB_BATCH_FAILED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.INIT_SUCCESS_SUB_BATCHES; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REMAINING_SUB_BATCH; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.REQUEST_ID; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SPLITTING_ENABLED; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SUB_BATCHES; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 54f5232b..a78393de 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -51,7 +51,7 @@ mock-payment-schema: authorization: "/batches/" channel: - hostname: "http://ph-ee-connector-channel:82" + hostname: "https://channel.mifos.gazelle.localhost" cloud: aws: @@ -73,8 +73,6 @@ cloud: tenants: "greenbank, bluebank" - - #payment mode moved to a separate application property file for helm rewrite config: @@ -136,9 +134,9 @@ server: security: jws: - enable: true + enable: false response: - enable: true + enable: false identity_account_mapper: hostname : "http://ph-ee-identity-account-mapper:80" @@ -151,18 +149,17 @@ bulk_processor: hostname : "http://ph-ee-connector-bulk:82" csv: - columnNames: "id,request_id,payment_mode,payer_identifier_type,payer_identifier,payee_identifier_type,payee_identifier,amount,currency,note,account_number,program_shortcode,cycle,payee_dfsp_id,batch_id" + columnNames: "id,request_id,payment_mode,payer_identifier_type,payer_identifier,payee_identifier_type,payee_identifier,amount,currency,note" size : 100000 # in bytes budget-account: registeringInstitutions: - - id: "123" + - id: "greenbank" programs: - id: "SocialWelfare" name: "Social Welfare" identifierType: "ACCOUNT" - identifierValue: "123456789" - + identifierValue: "1" management: endpoint: @@ -213,7 +210,8 @@ gov-stack-client: header-key: "X-GovStack-Client" header-value: "PAYMENT-BB" - +# TD Dec 2025 bpmn.flows looks like it is not actually used by bulk-processor +# so commenting out but leaving for now as bpmn: flows: payment-transfer: "PayerFundTransfer-{dfspid}" @@ -227,7 +225,8 @@ bpmn: international-remittance-payee: "international_remittance_payee_process-{dfspid}" international-remittance-payer: "international_remittance_payer_process-{dfspid}" debit-party-process: "debit_party_process-{dfspid}" - bulk-processor: "bulk_processor-{dfspid}" + #bulk-processor: "bulk_processor-{dfspid}" + bulk-processor: "bulk_processor_account_lookup-DFSPID.bpmn" slcb: "slcb-{dfspid}" bpmns: @@ -235,8 +234,8 @@ bpmns: - id: "greenbank" flows: payment-transfer: "minimal_mock_fund_transfer-{dfspid}" - batch-transactions: "bulk_processor-{dfspid}" - fred-transactions: "bulk_processor_account_lookup-{dfspid}" + #batch-transactions: "bulk_processor-{dfspid}" + batch-transactions: "bulk_processor_account_lookup-{dfspid}" - id: "bluebank" flows: batch-transactions: "bulk_processor-{dfspid}" From c902e3fdf92f5b0dad1974c61a09d68e42be36e2 Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Thu, 8 Jan 2026 01:51:46 +0000 Subject: [PATCH 146/156] correcting workflow settings in application.yaml --- src/main/resources/application.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index a78393de..2d14a2c2 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -234,8 +234,8 @@ bpmns: - id: "greenbank" flows: payment-transfer: "minimal_mock_fund_transfer-{dfspid}" - #batch-transactions: "bulk_processor-{dfspid}" - batch-transactions: "bulk_processor_account_lookup-{dfspid}" + batch-transactions: "bulk_processor-{dfspid}" + #batch-transactions: "bulk_processor_account_lookup-{dfspid}" - id: "bluebank" flows: batch-transactions: "bulk_processor-{dfspid}" From 60142d694c52b7a3ee9ff30c0cf33a9cd949c3ce Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Thu, 8 Jan 2026 23:41:32 +0000 Subject: [PATCH 147/156] update for redbank tenant and closedloop workflow --- .circleci/config.yml | 2 +- src/main/resources/application.yaml | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 16f0e510..7d24e1bf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -183,4 +183,4 @@ workflows: only: - main # Or your primary branch name like 'master' context: - - DOCKER \ No newline at end of file + - DOCKER diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 2d14a2c2..c2bd8c62 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -71,7 +71,7 @@ cloud: connection-string: -tenants: "greenbank, bluebank" +tenants: "greenbank, bluebank, redbank" #payment mode moved to a separate application property file for helm rewrite @@ -186,7 +186,6 @@ payment-mode: - id: "CLOSEDLOOP" type: "BULK" endpoint: "bulk_connector_{MODE}-{dfspid}" - debulkingDfspid: "greenbank" batch-authorization: callback-url: "${BULK_PROCESSOR_CALLBACK_URL:https://bulk-processor.mifos.gazelle.test/authorization/callback}" @@ -236,6 +235,10 @@ bpmns: payment-transfer: "minimal_mock_fund_transfer-{dfspid}" batch-transactions: "bulk_processor-{dfspid}" #batch-transactions: "bulk_processor_account_lookup-{dfspid}" + - id: "redbank" + flows: + payment-transfer: "minimal_mock_fund_transfer-{dfspid}" + batch-transactions: "bulk_processor-{dfspid}" - id: "bluebank" flows: batch-transactions: "bulk_processor-{dfspid}" From 91519b6dbf0e681782ae0920610e580f8b518c76 Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Mon, 19 Jan 2026 03:46:43 +0000 Subject: [PATCH 148/156] updates for mergeback --- .../org/mifos/processor/bulk/camel/routes/Routes.java | 10 +++++++++- .../processor/bulk/zeebe/worker/MergeBackWorker.java | 6 ++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java index de537c0b..3c619433 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java @@ -1,9 +1,12 @@ package org.mifos.processor.bulk.camel.routes; +import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID; import static org.mifos.processor.bulk.camel.config.CamelProperties.IS_BATCH_READY; +import static org.mifos.processor.bulk.camel.config.CamelProperties.OPS_APP_ACCESS_TOKEN; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.BATCH_ID; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.IS_SAMPLE_READY; import static org.mifos.processor.bulk.zeebe.ZeebeVariables.SAMPLED_TX_IDS; +import static org.mifos.processor.bulk.zeebe.ZeebeVariables.TENANT_ID; import com.google.gson.Gson; import java.util.ArrayList; @@ -30,6 +33,8 @@ private void routeCheckTransactions() { String id = "check-transactions"; from("direct:" + id).id(id).log("Fetching transaction details") // set request params + .setHeader("Authorization", simple("Bearer ${exchangeProperty." + OPS_APP_ACCESS_TOKEN + "}")) + .setHeader(HEADER_PLATFORM_TENANT_ID, simple("${exchangeProperty." + TENANT_ID + "}")) .toD(operationsAppConfig.batchTransactionEndpoint).process(exchange -> { // get response body JSONObject transfers = new JSONObject(exchange.getIn().getBody(String.class)); @@ -61,7 +66,10 @@ private void routeSampleTransactions() { String id = "sample-transactions"; from("direct:" + id).id(id).log("Fetching transaction details").process(exchange -> { exchange.getIn().setHeader("batchId", exchange.getProperty(BATCH_ID)); - }).toD(operationsAppConfig.batchTransactionEndpoint).process(exchange -> { + }) + .setHeader("Authorization", simple("Bearer ${exchangeProperty." + OPS_APP_ACCESS_TOKEN + "}")) + .setHeader(HEADER_PLATFORM_TENANT_ID, simple("${exchangeProperty." + TENANT_ID + "}")) + .toD(operationsAppConfig.batchTransactionEndpoint).process(exchange -> { // get response body // check if batch is ready for sampling diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java index fd55824f..f5ec54de 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/MergeBackWorker.java @@ -61,9 +61,11 @@ public void setup() { sendToCamelRoute(RouteId.MERGE_BACK, exchange); - boolean mergeCompleted = exchange.getProperty(MERGE_COMPLETED, Boolean.class); + Boolean mergeCompletedObj = exchange.getProperty(MERGE_COMPLETED, Boolean.class); + boolean mergeCompleted = mergeCompletedObj != null ? mergeCompletedObj : false; if (mergeCompleted) { - variables.put(MERGE_FAILED, exchange.getProperty(MERGE_FAILED, Boolean.class)); + Boolean mergeFailedObj = exchange.getProperty(MERGE_FAILED, Boolean.class); + variables.put(MERGE_FAILED, mergeFailedObj != null ? mergeFailedObj : false); String resultFile = exchange.getProperty(RESULT_FILE, String.class); if (resultFile != null && !resultFile.isEmpty()) { variables.put(RESULT_FILE, resultFile); From d99c84434befcf457bb6249e4345375a75ea9183 Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Mon, 19 Jan 2026 03:51:17 +0000 Subject: [PATCH 149/156] mods for selecting workflow at runtime --- .../bulk/connectors/service/ProcessorStartRouteService.java | 6 +++++- src/main/resources/application.yaml | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java index dc02e636..d936546c 100644 --- a/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java +++ b/src/main/java/org/mifos/processor/bulk/connectors/service/ProcessorStartRouteService.java @@ -273,7 +273,11 @@ public void startBatchProcessCsv(Exchange exchange) throws IOException { logger.info("Variables published to zeebe: {}", variables); JSONObject response = new JSONObject(); - String bpmn = processorStartRoute.getWorkflowForTenant(exchange.getProperty(TENANT_NAME).toString(), "batch-transactions"); + // Check if GovStack mode (registeringInstituteId is set) to select appropriate workflow + String registeringInstituteId = exchange.getProperty(REGISTERING_INSTITUTE_ID, String.class); + String flowType = StringUtils.hasText(registeringInstituteId) ? "batch-transactions-govstack" : "batch-transactions"; + logger.info("Selecting workflow flow type: {} (registeringInstituteId: {})", flowType, registeringInstituteId); + String bpmn = processorStartRoute.getWorkflowForTenant(exchange.getProperty(TENANT_NAME).toString(), flowType); try { logger.info("FREDa "); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index c2bd8c62..27060d2a 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -51,7 +51,7 @@ mock-payment-schema: authorization: "/batches/" channel: - hostname: "https://channel.mifos.gazelle.localhost" + hostname: "https://ph-ee-connector-channel:8443" cloud: aws: @@ -234,7 +234,7 @@ bpmns: flows: payment-transfer: "minimal_mock_fund_transfer-{dfspid}" batch-transactions: "bulk_processor-{dfspid}" - #batch-transactions: "bulk_processor_account_lookup-{dfspid}" + batch-transactions-govstack: "bulk_processor_account_lookup-{dfspid}" - id: "redbank" flows: payment-transfer: "minimal_mock_fund_transfer-{dfspid}" From bd3fac9ba93c05d446f47588131668764045ceb8 Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Mon, 19 Jan 2026 22:41:49 +0000 Subject: [PATCH 150/156] bugfixes for govstack mode --- .../routes/AccountLookupCallbackRoute.java | 6 +- .../processor/bulk/camel/routes/Routes.java | 57 +++++++++---------- src/main/resources/application.yaml | 5 +- 3 files changed, 35 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupCallbackRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupCallbackRoute.java index 9b4a3c28..1232ef7e 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupCallbackRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupCallbackRoute.java @@ -91,8 +91,10 @@ public List updateTransactionStatus(List transac } catch (NumberFormatException e) { logger.error(e.getMessage()); } - String identifier = matchingBeneficiary.get().getFinancialAddress(); - transaction.setPayeeIdentifier(identifier); + // Store financialAddress in accountNumber field for reconciliation + // DO NOT overwrite payeeIdentifier - keep the MSISDN for party lookup! + String financialAddress = matchingBeneficiary.get().getFinancialAddress(); + transaction.setAccountNumber(financialAddress); transaction.setPayeeDfspId(matchingBeneficiary.get().getBankingInstitutionCode()); updatedTransactionList.add(transaction); } else { diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java index 3c619433..191b7361 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/Routes.java @@ -66,39 +66,38 @@ private void routeSampleTransactions() { String id = "sample-transactions"; from("direct:" + id).id(id).log("Fetching transaction details").process(exchange -> { exchange.getIn().setHeader("batchId", exchange.getProperty(BATCH_ID)); - }) - .setHeader("Authorization", simple("Bearer ${exchangeProperty." + OPS_APP_ACCESS_TOKEN + "}")) + }).setHeader("Authorization", simple("Bearer ${exchangeProperty." + OPS_APP_ACCESS_TOKEN + "}")) .setHeader(HEADER_PLATFORM_TENANT_ID, simple("${exchangeProperty." + TENANT_ID + "}")) .toD(operationsAppConfig.batchTransactionEndpoint).process(exchange -> { - // get response body + // get response body - // check if batch is ready for sampling - if (exchange.getProperty(IS_BATCH_READY, String.class).equals("false")) { - return; - } - // sample transactions - JSONObject transfers = new JSONObject(exchange.getIn().getBody(String.class)); - final ArrayList successfulTransactionIds = new ArrayList<>(); - final ArrayList sampledTransactionIds = new ArrayList<>(); - for (Iterator it = transfers.keys(); it.hasNext();) { - String transactionId = it.next(); - String transactionStatus = transfers.getString(transactionId); - if (transactionStatus.equals("COMPLETED")) { - successfulTransactionIds.add(transactionId); - } - } - Collections.shuffle(successfulTransactionIds); - int sampleSize = (int) (successfulTransactionIds.size() * 0.9); - for (int i = 0; i < sampleSize; i++) { - sampledTransactionIds.add(successfulTransactionIds.get(i)); - } - HashMap newVariables = new HashMap<>(); - newVariables.put(SAMPLED_TX_IDS, new Gson().toJson(sampledTransactionIds)); + // check if batch is ready for sampling + if (exchange.getProperty(IS_BATCH_READY, String.class).equals("false")) { + return; + } + // sample transactions + JSONObject transfers = new JSONObject(exchange.getIn().getBody(String.class)); + final ArrayList successfulTransactionIds = new ArrayList<>(); + final ArrayList sampledTransactionIds = new ArrayList<>(); + for (Iterator it = transfers.keys(); it.hasNext();) { + String transactionId = it.next(); + String transactionStatus = transfers.getString(transactionId); + if (transactionStatus.equals("COMPLETED")) { + successfulTransactionIds.add(transactionId); + } + } + Collections.shuffle(successfulTransactionIds); + int sampleSize = (int) (successfulTransactionIds.size() * 0.9); + for (int i = 0; i < sampleSize; i++) { + sampledTransactionIds.add(successfulTransactionIds.get(i)); + } + HashMap newVariables = new HashMap<>(); + newVariables.put(SAMPLED_TX_IDS, new Gson().toJson(sampledTransactionIds)); - // store the sampled transaction ids in zeebe variable - zeebeClient.newSetVariablesCommand(Long.parseLong(exchange.getProperty(BATCH_ID).toString())).variables(newVariables).send() - .join(); + // store the sampled transaction ids in zeebe variable + zeebeClient.newSetVariablesCommand(Long.parseLong(exchange.getProperty(BATCH_ID).toString())).variables(newVariables) + .send().join(); - }); + }); } } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 27060d2a..ff9fc5c6 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -158,8 +158,8 @@ budget-account: programs: - id: "SocialWelfare" name: "Social Welfare" - identifierType: "ACCOUNT" - identifierValue: "1" + identifierType: "MSISDN" + identifierValue: "0413509790" management: endpoint: @@ -239,6 +239,7 @@ bpmns: flows: payment-transfer: "minimal_mock_fund_transfer-{dfspid}" batch-transactions: "bulk_processor-{dfspid}" + batch-transactions-govstack: "bulk_processor_account_lookup-{dfspid}" - id: "bluebank" flows: batch-transactions: "bulk_processor-{dfspid}" From 8a689ade21f66166360d75af92bac5fe4ce81538 Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Tue, 20 Jan 2026 05:51:29 +0000 Subject: [PATCH 151/156] updated build version to 2.0.0.mifos-SNAPSHOT --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index e0c57f05..3d52e720 100644 --- a/build.gradle +++ b/build.gradle @@ -224,7 +224,7 @@ configure(this) { group = 'org.mifos' -version = 'gazelle-1.1.0' +version = '2.0.0.mifos-SNAPSHOT' description = 'ph-ee-processor-bulk' sourceCompatibility = '17' From 44acb61ed4d5b7ad0fcd0716cae4af682a381f3d Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Wed, 21 Jan 2026 23:05:30 +0000 Subject: [PATCH 152/156] remove check for existing image -- for now --- .circleci/config.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7d24e1bf..810e7718 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,20 +36,20 @@ jobs: apt-cache madison docker-ce-cli | grep 20.10 apt-get install -y docker-ce-cli=5:20.10.24~3-0~debian-bullseye - docker-buildx/install - - run: - name: Check if Docker image tag exists - command: | - IMAGE_TAG=$CIRCLE_TAG - # Using lowercase variables defined above - IMAGE_NAME="$DOCKER_ORG_LOWER/$DOCKER_REPO_LOWER" - echo "Checking for Docker image: $IMAGE_NAME:$IMAGE_TAG" - # Ensure DOCKERHUB_USERNAME and DOCKERHUB_PASSWORD are set in your CircleCI Context or Project Environment Variables - if curl -s -f -u "$DOCKERHUB_USERNAME":"$DOCKERHUB_PASSWORD" "https://hub.docker.com/v2/repositories/$IMAGE_NAME/tags/$IMAGE_TAG" > /dev/null; then - echo "Skipping the build and push as the tag $IMAGE_TAG already exists in Docker Hub for image $IMAGE_NAME." - circleci-agent step halt - else - echo "Tag $IMAGE_TAG does not exist for image $IMAGE_NAME. Proceeding with build." - fi + # - run: + # name: Check if Docker image tag exists + # command: | + # IMAGE_TAG=$CIRCLE_TAG + # # Using lowercase variables defined above + # IMAGE_NAME="$DOCKER_ORG_LOWER/$DOCKER_REPO_LOWER" + # echo "Checking for Docker image: $IMAGE_NAME:$IMAGE_TAG" + # # Ensure DOCKERHUB_USERNAME and DOCKERHUB_PASSWORD are set in your CircleCI Context or Project Environment Variables + # if curl -s -f -u "$DOCKERHUB_USERNAME":"$DOCKERHUB_PASSWORD" "https://hub.docker.com/v2/repositories/$IMAGE_NAME/tags/$IMAGE_TAG" > /dev/null; then + # echo "Skipping the build and push as the tag $IMAGE_TAG already exists in Docker Hub for image $IMAGE_NAME." + # circleci-agent step halt + # else + # echo "Tag $IMAGE_TAG does not exist for image $IMAGE_NAME. Proceeding with build." + # fi - run: name: Build Application command: ./gradlew bootJar From 84b0f476e0a6ef872ac70464820ee34ff7c0e04b Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Thu, 22 Jan 2026 06:17:41 +0000 Subject: [PATCH 153/156] making certs handling consistent and removing baked in files from docker image --- .gitignore | 6 +++++- src/main/resources/application.yaml | 6 +++--- src/main/resources/keystore.jks | Bin 2272 -> 0 bytes 3 files changed, 8 insertions(+), 4 deletions(-) delete mode 100644 src/main/resources/keystore.jks diff --git a/.gitignore b/.gitignore index 14cce966..3d34106d 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,8 @@ build/ .vscode/ .DS_Store -/.mvn \ No newline at end of file +/.mvn +# Certificate files (use K8s secrets instead) +*.jks +*.p12 +*.keystore diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index ff9fc5c6..62cf3129 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -125,10 +125,10 @@ callback-phases: server: ssl: key-alias: "tomcat-https" - key-store: "classpath:keystore.jks" + key-store: "file:/tls/keystore.jks" key-store-type: JKS - key-password: "password" - key-store-password: "password" + key-password: "changeit" + key-store-password: "changeit" port: 8443 # HTTPS port for Spring Boot (external traffic, gRPC) diff --git a/src/main/resources/keystore.jks b/src/main/resources/keystore.jks deleted file mode 100644 index f0040c54511f5db99680b8393ef6111a2b4ca7f3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2272 zcmc(g`8U*!7sux_iw0vIWDI4^R*Yp#b|Y(M2BVREjYefDLL{1mWSeA7*<~q1h7j53 zpvUtdMG;BXtWDXz;n8!x-*cY-;QPb<;q^ZEo^#Lp-gEE%%Ki!n1Oguh@J}E@0{y)` zLrzgcLV|)1#rV%OQA-dA24HB=Jq#}gmof(c1(d)70009)Y0$-~Pl`Y6;IRu5!r9D) z;nu5KQL`@3Z+b~XixlUE2UqUc}tM_MKt9Mx@9W(~hX!m?nKG_m!SR0d$t%%_p-b zezecL)0s2OUs-dq19cG#W18n)Y?jnTNiF|hnFUKD9*x%`lxl$-FX5%6V z+g1^#H-#Mp`?&wbNzVYj;nn(BvDE`DO;yAqn&qQWL*|j@XE)DU-i8Q;>hOjaF*sWx zdb?OA;h;2}UX`VzIQgL}T=FN)UfD24qE=pvi#aj+<#_xJxD%=@l^)5_?vpS=z2@Jg zJ9nDmALj3tN{PNoVWZYZW|>D(+s?v?ExeibTy<^5$5_}B8Gr74KmMUfKRYpkQ?B>0 zj#NtP^78gVI!bHaqQ^@JZ{vXSiMy5Q=VqkmGwpI7a zjJ|X`U%R*m>SCmgaXwqcN)?UVj`9(luY z$lZIbkX!t=>V0y`>p&jAD)BRsfsKntRyPpw{Yf-mn^_BLUq{ciB3J2_yEAUNpL*XC zx{0xqNYRJ8MybIcd`D#Z_xc@tjLuv%zcgGE;!a8~uH=_RDc5-3JJrqh`RKBIMy8|6 zaB6z`XQVuO>CQ7v(+gYG;*td_XTM@P<*++wwvbEJTE#1k%;KrGpx4n+J^{s!Ghs#Z z*%h-14Eu}r`olF*Szb_Wn9yWsfiQ2WD?BvL@p5hkPVR+kbasp7`HeWI^CvQV`2PPYiBi)&4QoQ+Ab6yw`_Qcx>+POVG>p_>~sy-o8_t{LguWIjCSU z+8_|*0fq)i!O*~=1rRU*26KYO`zbkA#3Tz04+p1GoJHIg4*+n(K$z=l+!#&{2nEW4 z1Y45S1Tp--0gmLf!jnn(b9gKK|Fk1WPGeJ&Ii5fvQ`7`7e82O!kZ?0c5}s^!SSo}O z{4H@KxlK&%O$pXEL{ri^HDL_$w+cZb@b;EglID1lIfdYeId+ssT?2!~Xkc*aIE=Q_ zA!;6?`hW5N)fNqq{r?^tr~%!+r*RZ{aYj$~5FEs@aU%N{>nDslU@7 zsEyOSSZ6U|uwZSBTgFR17@Kv;CM1!nc>R_`7eTe5GBN)Q*%Ir``&ZtP#jQs9w$x3Q zg*(E<#iVvrnLIg|ijo#xMgU{{O|k1K+GL^yQ_SGRNeh{_OZW$_2KTnAo74FGP^^Q5 zJ*3(#15TX-SAB=&Y|wwIeBc>gB1E^jQ(y=H0*a+D$1xIzQ?rImY5_AC(|a z>Sp^4z|5uVzK?hQ+r+~yJ2VjhX;kP4Z8WDv7ky)D%@{6;;nIyqm7^n9z{}DNwH5ZJ zq6WXh?!4c$sjR8X>m^Aoj+1n`J<7=-#g#JKK9nYlUk5?mHrF zQJ1$fOmIS3D_J#H#E2axJLT2Eksy&(x=N<*z*bv0y%bH#8ZHx)d-6p+Hxrzwny<`S zY{mNNip33zx~FSP_x@o#pE2QtXXw>D>_D)G1I*TYeY_094Yh}M%Ub-N9I&6N@8+bO zGCyf(@1&((>7*LFVm@fgH`1*cjOI9+Z5O67!}avq4YRo*HKDzJvz+_19M{ n@or^P5_}!D?Ip(*1Yg&p=qaYQu$3FMnWn&u$dr1#Uv Date: Fri, 23 Jan 2026 00:14:51 +0000 Subject: [PATCH 154/156] move keystore from jks to pcs --- src/main/resources/application.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 62cf3129..c45be209 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -125,8 +125,8 @@ callback-phases: server: ssl: key-alias: "tomcat-https" - key-store: "file:/tls/keystore.jks" - key-store-type: JKS + key-store: "file:/tls/keystore.p12" + key-store-type: PKCS12 key-password: "changeit" key-store-password: "changeit" port: 8443 # HTTPS port for Spring Boot (external traffic, gRPC) From 911916e5fe0cad6d558ba373cb917bd1dfebcede Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Wed, 11 Feb 2026 10:14:24 +0000 Subject: [PATCH 155/156] additions and mods for mastercard connector --- .../camel/processor/MastercardApiPayload.java | 31 +++++++++++++++++++ .../bulk/config/ExternalApiPayloadConfig.java | 6 ++++ src/main/resources/application.yaml | 15 ++++++--- 3 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/mifos/processor/bulk/camel/processor/MastercardApiPayload.java diff --git a/src/main/java/org/mifos/processor/bulk/camel/processor/MastercardApiPayload.java b/src/main/java/org/mifos/processor/bulk/camel/processor/MastercardApiPayload.java new file mode 100644 index 00000000..adf5c0d6 --- /dev/null +++ b/src/main/java/org/mifos/processor/bulk/camel/processor/MastercardApiPayload.java @@ -0,0 +1,31 @@ +package org.mifos.processor.bulk.camel.processor; + +import static org.mifos.processor.bulk.camel.config.CamelProperties.TRANSACTION_LIST_ELEMENT; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.function.Function; +import org.apache.camel.Exchange; +import org.mifos.connector.common.channel.dto.TransactionChannelRequestDTO; +import org.mifos.processor.bulk.schema.Transaction; +import org.mifos.processor.bulk.utility.Utils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class MastercardApiPayload implements Function { + + @Autowired + ObjectMapper objectMapper; + + @Override + public String apply(Exchange exchange) { + Transaction transaction = exchange.getProperty(TRANSACTION_LIST_ELEMENT, Transaction.class); + TransactionChannelRequestDTO inboundTransferPayload = Utils.convertTxnToInboundTransferPayload(transaction); + try { + return objectMapper.writeValueAsString(inboundTransferPayload); + } catch (JsonProcessingException e) { + throw new DTOJsonConversionException(MastercardApiPayload.class, "Unable to convert MastercardPayload to JSON", e); + } + } +} diff --git a/src/main/java/org/mifos/processor/bulk/config/ExternalApiPayloadConfig.java b/src/main/java/org/mifos/processor/bulk/config/ExternalApiPayloadConfig.java index 613e98d6..52f763d6 100644 --- a/src/main/java/org/mifos/processor/bulk/config/ExternalApiPayloadConfig.java +++ b/src/main/java/org/mifos/processor/bulk/config/ExternalApiPayloadConfig.java @@ -7,6 +7,7 @@ import lombok.Getter; import org.apache.camel.Exchange; import org.mifos.processor.bulk.camel.processor.GsmaApiPayload; +import org.mifos.processor.bulk.camel.processor.MastercardApiPayload; import org.mifos.processor.bulk.camel.processor.MojaloopApiPayload; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; @@ -23,6 +24,9 @@ public class ExternalApiPayloadConfig { @Autowired MojaloopApiPayload mojaloopApiPayload; + @Autowired + MastercardApiPayload mastercardApiPayload; + @Autowired PaymentModeConfiguration paymentModeConfiguration; @@ -33,6 +37,8 @@ private void registerApiProcessor() { payloadMap.put(paymentMode.getId(), gsmaApiPayload); } else if (paymentMode.getId().equalsIgnoreCase("mojaloop")) { payloadMap.put(paymentMode.getId(), mojaloopApiPayload); + } else if (paymentMode.getId().equalsIgnoreCase("MASTERCARD_CBS")) { + payloadMap.put(paymentMode.getId(), mastercardApiPayload); } } } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index c45be209..9a33c08f 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -56,13 +56,13 @@ channel: cloud: aws: enabled: true - s3BaseUrl: "https://s3.ap-south-1.amazonaws.com" + s3BaseUrl: "http://minio.paymenthub.svc.cluster.local:9000" credentials: - access-key: ${AWS_ACCESS_KEY:access_key_from_aws} - secret-key: ${AWS_SECRET_KEY:secret_key_from_aws} + access-key: ${AWS_ACCESS_KEY:root} + secret-key: ${AWS_SECRET_KEY:password} region: - static: ap-south-1 + static: us-east-1 stack: auto: false azure: @@ -186,6 +186,9 @@ payment-mode: - id: "CLOSEDLOOP" type: "BULK" endpoint: "bulk_connector_{MODE}-{dfspid}" + - id: "MASTERCARD_CBS" + type: "PAYMENT" + endpoint: "/channel/transfer" batch-authorization: callback-url: "${BULK_PROCESSOR_CALLBACK_URL:https://bulk-processor.mifos.gazelle.test/authorization/callback}" @@ -235,6 +238,10 @@ bpmns: payment-transfer: "minimal_mock_fund_transfer-{dfspid}" batch-transactions: "bulk_processor-{dfspid}" batch-transactions-govstack: "bulk_processor_account_lookup-{dfspid}" + - id: "greenbank-mastercard" + flows: + payment-transfer: "MastercardFundTransfer-{dfspid}" + batch-transactions: "bulk_processor_account_lookup-{dfspid}" - id: "redbank" flows: payment-transfer: "minimal_mock_fund_transfer-{dfspid}" From d0164289107a00ee50577e2b9fc51e7620f986c0 Mon Sep 17 00:00:00 2001 From: Tom Daly Date: Fri, 20 Feb 2026 03:01:41 +0000 Subject: [PATCH 156/156] adding logging for accountmapper calls --- .../processor/bulk/camel/routes/AccountLookupRoute.java | 7 +++++++ .../bulk/zeebe/worker/BatchAccountLookupWorker.java | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java index 79ffce9f..c60454f7 100644 --- a/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java +++ b/src/main/java/org/mifos/processor/bulk/camel/routes/AccountLookupRoute.java @@ -71,15 +71,22 @@ public void process(Exchange exchange) throws Exception { List transactionList = exchange.getProperty(TRANSACTION_LIST, List.class); HashMap> stringListHashMap = new HashMap<>(); List beneficiaryDTOList = new ArrayList<>(); + logger.info("=== ACCOUNT LOOKUP ROUTE DEBUG ==="); + logger.info("Building beneficiary list from {} transactions", transactionList != null ? transactionList.size() : 0); transactionList.forEach(transaction -> { + logger.info("Adding beneficiary - payeeIdentity: {}, paymentMode: {}", transaction.getPayeeIdentifier(), + transaction.getPaymentMode()); beneficiaryDTOList.add(new BeneficiaryDTO(transaction.getPayeeIdentifier(), "", "", "")); }); String requestId = exchange.getProperty(REQUEST_ID, String.class); String callbackUrl = exchange.getProperty(CALLBACK, String.class); String registeringInstitutionId = exchange.getProperty(HEADER_REGISTERING_INSTITUTE_ID, String.class); + logger.info("Request metadata - ID: {}, RegisteringInstitution: {}, Callback: {}", requestId, registeringInstitutionId, + callbackUrl); AccountMapperRequestDTO accountMapperRequestDTO = new AccountMapperRequestDTO(requestId, registeringInstitutionId, beneficiaryDTOList); String requestBody = objectMapper.writeValueAsString(accountMapperRequestDTO); + logger.info("Request body to send to identity mapper ({} chars): {}", requestBody.length(), requestBody); exchange.getIn().setHeader(CALLBACK, callbackUrl); exchange.getIn().setHeader(REGISTERING_INSTITUTION_ID, registeringInstitutionId); diff --git a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupWorker.java b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupWorker.java index f1cc8c6d..d86d535f 100644 --- a/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupWorker.java +++ b/src/main/java/org/mifos/processor/bulk/zeebe/worker/BatchAccountLookupWorker.java @@ -59,8 +59,15 @@ public void setup() { exchange.setProperty(CALLBACK, identityMapperURL + batchAccountLookupCallback); try { + logger.info("=== BATCH ACCOUNT LOOKUP WORKER DEBUG ==="); + logger.info("Sending to ACCOUNT_LOOKUP route with registeringInstituteId: {}", registeringInstituteId); + logger.info("Filename: {}", filename); + logger.info("Callback URL: {}", identityMapperURL + batchAccountLookupCallback); sendToCamelRoute(RouteId.ACCOUNT_LOOKUP, exchange); + logger.info("ACCOUNT_LOOKUP route call completed successfully"); } catch (Exception e) { + logger.error("=== ACCOUNT LOOKUP FAILED ==="); + logger.error("Exception calling ACCOUNT_LOOKUP route: " + e.getMessage(), e); variables.put(PARTY_LOOKUP_FAILED, true); } client.newCompleteCommand(job.getKey()).variables(variables).send();