support for deletedRetention config at table config level

owasp-suppressions.xml

@@ -2,7 +2,7 @@
 <suppressions xmlns="https://jeremylong.github.io/DependencyCheck/dependency-suppression.1.3.xsd">
   <suppress>
     <notes><![CDATA[
-   Any hypertrace dep
+   Hypertrace dependencies identified as some TPLs with vulnerabilities
    ]]></notes>
     <packageUrl regex="true">^pkg:maven/org\.hypertrace\..*@.*$</packageUrl>
     <cpe>cpe:/a:grpc:grpc</cpe>
@@ -11,7 +11,7 @@
   </suppress>
   <suppress>
     <notes><![CDATA[
-    Pinot mismatches
+    Pinot packages identified as some TPLs with vulnerabilities
    ]]></notes>
     <packageUrl regex="true">^pkg:maven/org\.apache\.pinot/.*@.*$</packageUrl>
     <cpe>cpe:/a:apache:avro</cpe>
@@ -34,34 +34,9 @@
   </suppress>
   <suppress>
     <notes><![CDATA[
-   Pinot packages being matched as Avro
+   Pinot packages being falsely matched as Avro
    ]]></notes>
     <packageUrl regex="true">^pkg:maven/org\.apache\.pinot/.*@.*$</packageUrl>
     <cpe>cpe:/a:avro_project:avro</cpe>
   </suppress>
-  <suppress>
-    <notes><![CDATA[
-    This CVE impacts the maven build, not the runtime - ref https://spark.apache.org/security.html#CVE-2018-11804
-   ]]></notes>
-    <packageUrl regex="true">^pkg:maven/org\.apache\.spark/spark\-.*@.*$</packageUrl>
-    <cve>CVE-2018-11804</cve>
-  </suppress>
-  <suppress>
-    <notes><![CDATA[
-    These CVEs impacts spark as standalone master, not being used in that way here - refs:
-     https://spark.apache.org/security.html#CVE-2018-11770
-     https://spark.apache.org/security.html#CVE-2018-17190
-   ]]></notes>
-    <packageUrl regex="true">^pkg:maven/org\.apache\.spark/spark\-.*@.*$</packageUrl>
-    <cve>CVE-2018-11770</cve>
-    <cve>CVE-2018-17190</cve>
-  </suppress>
-  <suppress>
-    <notes><![CDATA[
-    Dep removed in favor of newer apache http client in pinot 1.0
-   ]]></notes>
-    <packageUrl regex="true">^pkg:maven/commons\-httpclient/commons\-httpclient@.*$</packageUrl>
-    <cve>CVE-2012-5783</cve>
-    <cve>CVE-2020-13956</cve>
-  </suppress>
 </suppressions>

view-creator-framework/build.gradle.kts

@@ -17,7 +17,7 @@ dependencies {
   api("org.apache.commons:commons-compress:1.26.0") {
     because("https://www.tenable.com/cve/CVE-2024-25710")
   }
-  implementation("org.apache.pinot:pinot-tools:0.12.1") {
+  implementation("org.apache.pinot:pinot-tools:1.0.0") {
     // All these third party libraries are not used in view creation workflow.
     // They bring in lot of vulnerabilities (snyk). so, excluding unused libs
     exclude("com.google.protobuf", "protobuf-java")
@@ -29,16 +29,21 @@ dependencies {
     exclude("javax.servlet", "javax.servlet-api")
     exclude("org.apache.hadoop", "hadoop-common")
     exclude("org.apache.hadoop", "hadoop-hdfs")
+    exclude("org.apache.hadoop", "hadoop-hdfs-client")
     exclude("org.apache.helix", "helix-core")
     exclude("org.apache.httpcomponents", "httpclient")
     exclude("org.apache.kafka", "kafka_2.10")
     exclude("org.apache.logging.log4j", "log4j-core")
     exclude("org.apache.pinot", "pinot-controller")
     exclude("org.apache.pinot", "pinot-broker")
     exclude("org.apache.pinot", "pinot-kafka-2.0")
+    exclude("org.apache.pinot", "pinot-minion-builtin-tasks")
+    exclude("org.apache.pinot", "pinot-minion")
+    exclude("org.apache.pinot", "pinot-pulsar")
     exclude("org.apache.pinot", "pinot-parquet")
     exclude("org.apache.pinot", "pinot-server")
     exclude("org.apache.pinot", "pinot-s3")
+    exclude("org.apache.spark", "spark-launcher_2.12")
     exclude("org.apache.thrift", "libthrift")
     exclude("org.apache.zookeeper", "zookeeper")
     exclude("org.glassfish.jersey.containers", "jersey-container-grizzly2-http")
@@ -54,7 +59,6 @@ dependencies {
     implementation("org.apache.calcite:calcite-core:1.34.0")
     implementation("org.apache.calcite:calcite-babel:1.34.0")
     implementation("com.google.code.gson:gson:2.10.1")
-    implementation("org.apache.spark:spark-launcher_2.12:3.4.1")
     implementation("org.xerial.snappy:snappy-java:1.1.10.5")
     implementation("com.google.protobuf:protobuf-java-util:3.16.3")
     implementation("org.codehaus.janino:janino:3.1.11")

view-creator-framework/src/main/java/org/hypertrace/core/viewcreator/pinot/PinotTableSpec.java

@@ -40,6 +40,10 @@ public class PinotTableSpec {
   // Segments config
   private int numReplicas;
   @Optional private String replicasPerPartition = "1";
+  // Apache Pinot defaults this duration to 7 days, if not set explicitly.
+  // Using 1d as the default for all tables to keep the storage space under control for deleted
+  // segments
+  @Optional private String deletedSegmentsRetentionPeriod = "1d";
   private TimeUnit timeUnit;
   private String timeColumn;
   private String retentionTimeValue;

view-creator-framework/src/main/java/org/hypertrace/core/viewcreator/pinot/PinotUtils.java

@@ -377,6 +377,7 @@ public static TableConfig buildPinotTableConfig(
             .setNumReplicas(pinotTableSpec.getNumReplicas())
             .setRetentionTimeValue(pinotTableSpec.getRetentionTimeValue())
             .setRetentionTimeUnit(pinotTableSpec.getRetentionTimeUnit())
+            .setDeletedSegmentsRetentionPeriod(pinotTableSpec.getDeletedSegmentsRetentionPeriod())
             .setPeerSegmentDownloadScheme(pinotTableSpec.getPeerSegmentDownloadScheme())
             // Tenant configs
             .setBrokerTenant(pinotTableSpec.getBrokerTenant())
@@ -501,7 +502,11 @@ private static IngestionConfig toTableIngestionConfig(@Nullable PinotTableSpec t
       tableFilterConfig = new FilterConfig(filterConfig.getString(PINOT_FILTER_FUNCTION));
     }
 
-    return new IngestionConfig(null, null, tableFilterConfig, tableTransformConfigs, null, null);
+    IngestionConfig ingestionConfig = new IngestionConfig();
+    ingestionConfig.setFilterConfig(tableFilterConfig);
+    ingestionConfig.setTransformConfigs(tableTransformConfigs);
+
+    return ingestionConfig;
   }
 
   private static TagOverrideConfig toTagOverrideConfig(Config tenantTagOverrideConfig) {

view-creator-framework/src/test/java/org/hypertrace/core/viewcreator/pinot/PinotUtilsTest.java

@@ -210,6 +210,7 @@ public void testBuildRealtimeTableConfig() {
     assertEquals(1, tableConfig.getValidationConfig().getReplicasPerPartitionNumber());
     assertEquals("3", tableConfig.getValidationConfig().getRetentionTimeValue());
     assertEquals("DAYS", tableConfig.getValidationConfig().getRetentionTimeUnit());
+    assertEquals("1d", tableConfig.getValidationConfig().getDeletedSegmentsRetentionPeriod());
     assertEquals(
         "BalanceNumSegmentAssignmentStrategy",
         tableConfig.getValidationConfig().getSegmentAssignmentStrategy());