diff --git a/owasp-suppressions.xml b/owasp-suppressions.xml index d2e529a..feca0f7 100644 --- a/owasp-suppressions.xml +++ b/owasp-suppressions.xml @@ -2,7 +2,7 @@ ^pkg:maven/org\.hypertrace\..*@.*$ cpe:/a:grpc:grpc @@ -11,7 +11,7 @@ ^pkg:maven/org\.apache\.pinot/.*@.*$ cpe:/a:apache:avro @@ -34,34 +34,9 @@ ^pkg:maven/org\.apache\.pinot/.*@.*$ cpe:/a:avro_project:avro - - - ^pkg:maven/org\.apache\.spark/spark\-.*@.*$ - CVE-2018-11804 - - - - ^pkg:maven/org\.apache\.spark/spark\-.*@.*$ - CVE-2018-11770 - CVE-2018-17190 - - - - ^pkg:maven/commons\-httpclient/commons\-httpclient@.*$ - CVE-2012-5783 - CVE-2020-13956 - diff --git a/view-creator-framework/build.gradle.kts b/view-creator-framework/build.gradle.kts index 18d9199..0216aec 100644 --- a/view-creator-framework/build.gradle.kts +++ b/view-creator-framework/build.gradle.kts @@ -17,7 +17,7 @@ dependencies { api("org.apache.commons:commons-compress:1.26.0") { because("https://www.tenable.com/cve/CVE-2024-25710") } - implementation("org.apache.pinot:pinot-tools:0.12.1") { + implementation("org.apache.pinot:pinot-tools:1.0.0") { // All these third party libraries are not used in view creation workflow. // They bring in lot of vulnerabilities (snyk). so, excluding unused libs exclude("com.google.protobuf", "protobuf-java") @@ -29,6 +29,7 @@ dependencies { exclude("javax.servlet", "javax.servlet-api") exclude("org.apache.hadoop", "hadoop-common") exclude("org.apache.hadoop", "hadoop-hdfs") + exclude("org.apache.hadoop", "hadoop-hdfs-client") exclude("org.apache.helix", "helix-core") exclude("org.apache.httpcomponents", "httpclient") exclude("org.apache.kafka", "kafka_2.10") @@ -36,9 +37,13 @@ dependencies { exclude("org.apache.pinot", "pinot-controller") exclude("org.apache.pinot", "pinot-broker") exclude("org.apache.pinot", "pinot-kafka-2.0") + exclude("org.apache.pinot", "pinot-minion-builtin-tasks") + exclude("org.apache.pinot", "pinot-minion") + exclude("org.apache.pinot", "pinot-pulsar") exclude("org.apache.pinot", "pinot-parquet") exclude("org.apache.pinot", "pinot-server") exclude("org.apache.pinot", "pinot-s3") + exclude("org.apache.spark", "spark-launcher_2.12") exclude("org.apache.thrift", "libthrift") exclude("org.apache.zookeeper", "zookeeper") exclude("org.glassfish.jersey.containers", "jersey-container-grizzly2-http") @@ -54,7 +59,6 @@ dependencies { implementation("org.apache.calcite:calcite-core:1.34.0") implementation("org.apache.calcite:calcite-babel:1.34.0") implementation("com.google.code.gson:gson:2.10.1") - implementation("org.apache.spark:spark-launcher_2.12:3.4.1") implementation("org.xerial.snappy:snappy-java:1.1.10.5") implementation("com.google.protobuf:protobuf-java-util:3.16.3") implementation("org.codehaus.janino:janino:3.1.11") diff --git a/view-creator-framework/src/main/java/org/hypertrace/core/viewcreator/pinot/PinotTableSpec.java b/view-creator-framework/src/main/java/org/hypertrace/core/viewcreator/pinot/PinotTableSpec.java index 6ca475b..877c3e4 100644 --- a/view-creator-framework/src/main/java/org/hypertrace/core/viewcreator/pinot/PinotTableSpec.java +++ b/view-creator-framework/src/main/java/org/hypertrace/core/viewcreator/pinot/PinotTableSpec.java @@ -40,6 +40,10 @@ public class PinotTableSpec { // Segments config private int numReplicas; @Optional private String replicasPerPartition = "1"; + // Apache Pinot defaults this duration to 7 days, if not set explicitly. + // Using 1d as the default for all tables to keep the storage space under control for deleted + // segments + @Optional private String deletedSegmentsRetentionPeriod = "1d"; private TimeUnit timeUnit; private String timeColumn; private String retentionTimeValue; diff --git a/view-creator-framework/src/main/java/org/hypertrace/core/viewcreator/pinot/PinotUtils.java b/view-creator-framework/src/main/java/org/hypertrace/core/viewcreator/pinot/PinotUtils.java index b8c459c..e9173e0 100644 --- a/view-creator-framework/src/main/java/org/hypertrace/core/viewcreator/pinot/PinotUtils.java +++ b/view-creator-framework/src/main/java/org/hypertrace/core/viewcreator/pinot/PinotUtils.java @@ -377,6 +377,7 @@ public static TableConfig buildPinotTableConfig( .setNumReplicas(pinotTableSpec.getNumReplicas()) .setRetentionTimeValue(pinotTableSpec.getRetentionTimeValue()) .setRetentionTimeUnit(pinotTableSpec.getRetentionTimeUnit()) + .setDeletedSegmentsRetentionPeriod(pinotTableSpec.getDeletedSegmentsRetentionPeriod()) .setPeerSegmentDownloadScheme(pinotTableSpec.getPeerSegmentDownloadScheme()) // Tenant configs .setBrokerTenant(pinotTableSpec.getBrokerTenant()) @@ -501,7 +502,11 @@ private static IngestionConfig toTableIngestionConfig(@Nullable PinotTableSpec t tableFilterConfig = new FilterConfig(filterConfig.getString(PINOT_FILTER_FUNCTION)); } - return new IngestionConfig(null, null, tableFilterConfig, tableTransformConfigs, null, null); + IngestionConfig ingestionConfig = new IngestionConfig(); + ingestionConfig.setFilterConfig(tableFilterConfig); + ingestionConfig.setTransformConfigs(tableTransformConfigs); + + return ingestionConfig; } private static TagOverrideConfig toTagOverrideConfig(Config tenantTagOverrideConfig) { diff --git a/view-creator-framework/src/test/java/org/hypertrace/core/viewcreator/pinot/PinotUtilsTest.java b/view-creator-framework/src/test/java/org/hypertrace/core/viewcreator/pinot/PinotUtilsTest.java index 7445bac..e5d95db 100644 --- a/view-creator-framework/src/test/java/org/hypertrace/core/viewcreator/pinot/PinotUtilsTest.java +++ b/view-creator-framework/src/test/java/org/hypertrace/core/viewcreator/pinot/PinotUtilsTest.java @@ -210,6 +210,7 @@ public void testBuildRealtimeTableConfig() { assertEquals(1, tableConfig.getValidationConfig().getReplicasPerPartitionNumber()); assertEquals("3", tableConfig.getValidationConfig().getRetentionTimeValue()); assertEquals("DAYS", tableConfig.getValidationConfig().getRetentionTimeUnit()); + assertEquals("1d", tableConfig.getValidationConfig().getDeletedSegmentsRetentionPeriod()); assertEquals( "BalanceNumSegmentAssignmentStrategy", tableConfig.getValidationConfig().getSegmentAssignmentStrategy());