Skip to content

fix bug with NestedCommonFormatColumnFormatSpec default value resolution #18638

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
clintropolis merged 2 commits into from
Oct 16, 2025
Merged

fix bug with NestedCommonFormatColumnFormatSpec default value resolution #18638

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f8c75a0
fix bug with NestedCommonFormatColumnFormatSpec default value resolution
clintropolis Oct 15, 2025
e2060e2
adjustments
clintropolis Oct 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions processing/src/main/java/org/apache/druid/segment/IndexSpec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,9 +267,13 @@ public IndexSpec getEffectiveSpec()
}

if (autoColumnFormatSpec != null) {
bob.withAutoColumnFormatSpec(autoColumnFormatSpec.getEffectiveSpec(this));
bob.withAutoColumnFormatSpec(
NestedCommonFormatColumnFormatSpec.getEffectiveFormatSpec(autoColumnFormatSpec, this)
);
} else if (defaultSpec.autoColumnFormatSpec != null) {
bob.withAutoColumnFormatSpec(defaultSpec.autoColumnFormatSpec.getEffectiveSpec(this));
bob.withAutoColumnFormatSpec(
NestedCommonFormatColumnFormatSpec.getEffectiveFormatSpec(defaultSpec.autoColumnFormatSpec, this)
);
}

return bob.build();
Expand Down
190 changes: 93 additions & 97 deletions ...ing/src/main/java/org/apache/druid/segment/nested/NestedCommonFormatColumnFormatSpec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,9 @@
public class NestedCommonFormatColumnFormatSpec
{
private static final NestedCommonFormatColumnFormatSpec DEFAULT =
NestedCommonFormatColumnFormatSpec.builder()
.setObjectFieldsDictionaryEncoding(StringEncodingStrategy.UTF8_STRATEGY)
.setObjectStorageEncoding(ObjectStorageEncoding.SMILE)
.build();
builder().setObjectFieldsDictionaryEncoding(StringEncodingStrategy.UTF8_STRATEGY)
.setObjectStorageEncoding(ObjectStorageEncoding.SMILE)
.build();

public static Builder builder()
{
Expand All @@ -56,102 +55,27 @@ public static Builder builder(NestedCommonFormatColumnFormatSpec spec)
return new Builder(spec);
}

/**
* Create a {@link NestedCommonFormatColumnFormatSpec} with all fields fully populated. Values from the supplied
* column format spec take priority, any null values are then populated by checking
* {@link IndexSpec#getAutoColumnFormatSpec()}, then falling back to fields on {@link IndexSpec} itself if applicable,
* and finally resorting to hard coded defaults.
*/
public static NestedCommonFormatColumnFormatSpec getEffectiveFormatSpec(
@Nullable NestedCommonFormatColumnFormatSpec columnFormatSpec,
IndexSpec indexSpec
)
{
return Objects.requireNonNullElse(columnFormatSpec, DEFAULT).getEffectiveSpec(indexSpec);
}

@Nullable
private final StringEncodingStrategy objectFieldsDictionaryEncoding;
@Nullable
private final ObjectStorageEncoding objectStorageEncoding;
@Nullable
private final CompressionStrategy objectStorageCompression;
@Nullable
private final StringEncodingStrategy stringDictionaryEncoding;
@Nullable
private final CompressionStrategy dictionaryEncodedColumnCompression;
@Nullable
private final CompressionFactory.LongEncodingStrategy longColumnEncoding;
@Nullable
private final CompressionStrategy longColumnCompression;
@Nullable
private final CompressionStrategy doubleColumnCompression;
@Nullable
private final BitmapSerdeFactory bitmapEncoding;

@JsonCreator
public NestedCommonFormatColumnFormatSpec(
@JsonProperty("objectFieldsDictionaryEncoding") @Nullable StringEncodingStrategy objectFieldsDictionaryEncoding,
@JsonProperty("objectStorageEncoding") @Nullable ObjectStorageEncoding objectStorageEncoding,
@JsonProperty("objectStorageCompression") @Nullable CompressionStrategy objectStorageCompression,
@JsonProperty("stringDictionaryEncoding") @Nullable StringEncodingStrategy stringDictionaryEncoding,
@JsonProperty("dictionaryEncodedColumnCompression") @Nullable CompressionStrategy dictionaryEncodedColumnCompression,
@JsonProperty("longColumnEncoding") @Nullable CompressionFactory.LongEncodingStrategy longColumnEncoding,
@JsonProperty("longColumnCompression") @Nullable CompressionStrategy longColumnCompression,
@JsonProperty("doubleColumnCompression") @Nullable CompressionStrategy doubleColumnCompression
)
{
this(
objectFieldsDictionaryEncoding,
objectStorageEncoding,
objectStorageCompression,
stringDictionaryEncoding,
dictionaryEncodedColumnCompression,
longColumnEncoding,
longColumnCompression,
doubleColumnCompression,
null
);
}

/**
* Internal constructor used by {@link Builder} to set {@link #bitmapEncoding} during the process of resolving values
* for {@link #getEffectiveSpec(IndexSpec)}. {@link #bitmapEncoding} cannot vary per column, and is always set from
* {@link IndexSpec#getBitmapSerdeFactory()}.
*/
protected NestedCommonFormatColumnFormatSpec(
@Nullable StringEncodingStrategy objectFieldsDictionaryEncoding,
@Nullable ObjectStorageEncoding objectStorageEncoding,
@Nullable CompressionStrategy objectStorageCompression,
@Nullable StringEncodingStrategy stringDictionaryEncoding,
@Nullable CompressionStrategy dictionaryEncodedColumnCompression,
@Nullable CompressionFactory.LongEncodingStrategy longColumnEncoding,
@Nullable CompressionStrategy longColumnCompression,
@Nullable CompressionStrategy doubleColumnCompression,
@Nullable BitmapSerdeFactory bitmapEncoding
)
{
this.objectFieldsDictionaryEncoding = objectFieldsDictionaryEncoding;
this.objectStorageEncoding = objectStorageEncoding;
this.objectStorageCompression = objectStorageCompression;
this.stringDictionaryEncoding = stringDictionaryEncoding;
this.dictionaryEncodedColumnCompression = dictionaryEncodedColumnCompression;
this.longColumnEncoding = longColumnEncoding;
this.longColumnCompression = longColumnCompression;
this.doubleColumnCompression = doubleColumnCompression;
this.bitmapEncoding = bitmapEncoding;
}
final Builder builder = columnFormatSpec == null ? builder() : builder(columnFormatSpec);

/**
* Fully populate all fields of {@link NestedCommonFormatColumnFormatSpec}. Null values are populated first checking
* {@link IndexSpec#getAutoColumnFormatSpec()}, then falling back to fields on {@link IndexSpec} itself if applicable,
* and finally resorting to hard coded defaults.
*/
public NestedCommonFormatColumnFormatSpec getEffectiveSpec(IndexSpec indexSpec)
{
// this is a defensive check, the json spec can't set this, only the builder can
if (bitmapEncoding != null && !bitmapEncoding.equals(indexSpec.getBitmapSerdeFactory())) {
// this is a defensive check, the json spec of the column can't set this, only the builder can
if (builder.bitmapEncoding != null && !builder.bitmapEncoding.equals(indexSpec.getBitmapSerdeFactory())) {
throw new ISE(
"bitmapEncoding[%s] does not match indexSpec.bitmap[%s]",
bitmapEncoding,
builder.bitmapEncoding,
indexSpec.getBitmapSerdeFactory()
);
}
Builder builder = new Builder(this);
builder.setBitmapEncoding(indexSpec.getBitmapSerdeFactory());

final NestedCommonFormatColumnFormatSpec defaultSpec;
Expand All @@ -161,23 +85,23 @@ public NestedCommonFormatColumnFormatSpec getEffectiveSpec(IndexSpec indexSpec)
defaultSpec = DEFAULT;
}

if (objectFieldsDictionaryEncoding == null) {
if (builder.objectFieldsDictionaryEncoding == null) {
if (defaultSpec.getObjectFieldsDictionaryEncoding() != null) {
builder.setObjectFieldsDictionaryEncoding(defaultSpec.getObjectFieldsDictionaryEncoding());
} else {
builder.setObjectFieldsDictionaryEncoding(StringEncodingStrategy.DEFAULT);
}
}

if (objectStorageEncoding == null) {
if (builder.objectStorageEncoding == null) {
if (defaultSpec.getObjectStorageEncoding() != null) {
builder.setObjectStorageEncoding(defaultSpec.getObjectStorageEncoding());
} else {
builder.setObjectStorageEncoding(ObjectStorageEncoding.SMILE);
}
}

if (objectStorageCompression == null) {
if (builder.objectStorageCompression == null) {
if (defaultSpec.getObjectStorageCompression() != null) {
builder.setObjectStorageCompression(defaultSpec.getObjectStorageCompression());
} else if (indexSpec.getJsonCompression() != null) {
Expand All @@ -187,39 +111,39 @@ public NestedCommonFormatColumnFormatSpec getEffectiveSpec(IndexSpec indexSpec)
}
}

if (stringDictionaryEncoding == null) {
if (builder.stringDictionaryEncoding == null) {
if (defaultSpec.getStringDictionaryEncoding() != null) {
builder.setStringDictionaryEncoding(defaultSpec.getStringDictionaryEncoding());
} else {
builder.setStringDictionaryEncoding(indexSpec.getStringDictionaryEncoding());
}
}

if (dictionaryEncodedColumnCompression == null) {
if (builder.dictionaryEncodedColumnCompression == null) {
if (defaultSpec.getDictionaryEncodedColumnCompression() != null) {
builder.setDictionaryEncodedColumnCompression(defaultSpec.getDictionaryEncodedColumnCompression());
} else {
builder.setDictionaryEncodedColumnCompression(indexSpec.getDimensionCompression());
}
}

if (longColumnEncoding == null) {
if (builder.longColumnEncoding == null) {
if (defaultSpec.getLongColumnEncoding() != null) {
builder.setLongColumnEncoding(defaultSpec.getLongColumnEncoding());
} else {
builder.setLongColumnEncoding(indexSpec.getLongEncoding());
}
}

if (longColumnCompression == null) {
if (builder.longColumnCompression == null) {
if (defaultSpec.getLongColumnCompression() != null) {
builder.setLongColumnCompression(defaultSpec.getLongColumnCompression());
} else {
builder.setLongColumnCompression(indexSpec.getMetricCompression());
}
}

if (doubleColumnCompression == null) {
if (builder.doubleColumnCompression == null) {
if (defaultSpec.getDoubleColumnCompression() != null) {
builder.setDoubleColumnCompression(defaultSpec.getDoubleColumnCompression());
} else {
Expand All @@ -230,6 +154,78 @@ public NestedCommonFormatColumnFormatSpec getEffectiveSpec(IndexSpec indexSpec)
return builder.build();
}

@Nullable
private final StringEncodingStrategy objectFieldsDictionaryEncoding;
@Nullable
private final ObjectStorageEncoding objectStorageEncoding;
@Nullable
private final CompressionStrategy objectStorageCompression;
@Nullable
private final StringEncodingStrategy stringDictionaryEncoding;
@Nullable
private final CompressionStrategy dictionaryEncodedColumnCompression;
@Nullable
private final CompressionFactory.LongEncodingStrategy longColumnEncoding;
@Nullable
private final CompressionStrategy longColumnCompression;
@Nullable
private final CompressionStrategy doubleColumnCompression;
@Nullable
private final BitmapSerdeFactory bitmapEncoding;

@JsonCreator
public NestedCommonFormatColumnFormatSpec(
@JsonProperty("objectFieldsDictionaryEncoding") @Nullable StringEncodingStrategy objectFieldsDictionaryEncoding,
@JsonProperty("objectStorageEncoding") @Nullable ObjectStorageEncoding objectStorageEncoding,
@JsonProperty("objectStorageCompression") @Nullable CompressionStrategy objectStorageCompression,
@JsonProperty("stringDictionaryEncoding") @Nullable StringEncodingStrategy stringDictionaryEncoding,
@JsonProperty("dictionaryEncodedColumnCompression") @Nullable CompressionStrategy dictionaryEncodedColumnCompression,
@JsonProperty("longColumnEncoding") @Nullable CompressionFactory.LongEncodingStrategy longColumnEncoding,
@JsonProperty("longColumnCompression") @Nullable CompressionStrategy longColumnCompression,
@JsonProperty("doubleColumnCompression") @Nullable CompressionStrategy doubleColumnCompression
)
{
this(
objectFieldsDictionaryEncoding,
objectStorageEncoding,
objectStorageCompression,
stringDictionaryEncoding,
dictionaryEncodedColumnCompression,
longColumnEncoding,
longColumnCompression,
doubleColumnCompression,
null
);
}

/**
* Internal constructor used by {@link Builder} to set {@link #bitmapEncoding} during the process of resolving values
* for {@link #getEffectiveFormatSpec(NestedCommonFormatColumnFormatSpec, IndexSpec)}. {@link #bitmapEncoding} cannot
* vary per column, and is always set from {@link IndexSpec#getBitmapSerdeFactory()}.
*/
protected NestedCommonFormatColumnFormatSpec(
@Nullable StringEncodingStrategy objectFieldsDictionaryEncoding,
@Nullable ObjectStorageEncoding objectStorageEncoding,
@Nullable CompressionStrategy objectStorageCompression,
@Nullable StringEncodingStrategy stringDictionaryEncoding,
@Nullable CompressionStrategy dictionaryEncodedColumnCompression,
@Nullable CompressionFactory.LongEncodingStrategy longColumnEncoding,
@Nullable CompressionStrategy longColumnCompression,
@Nullable CompressionStrategy doubleColumnCompression,
@Nullable BitmapSerdeFactory bitmapEncoding
)
{
this.objectFieldsDictionaryEncoding = objectFieldsDictionaryEncoding;
this.objectStorageEncoding = objectStorageEncoding;
this.objectStorageCompression = objectStorageCompression;
this.stringDictionaryEncoding = stringDictionaryEncoding;
this.dictionaryEncodedColumnCompression = dictionaryEncodedColumnCompression;
this.longColumnEncoding = longColumnEncoding;
this.longColumnCompression = longColumnCompression;
this.doubleColumnCompression = doubleColumnCompression;
this.bitmapEncoding = bitmapEncoding;
}

@Nullable
@JsonProperty
public StringEncodingStrategy getObjectFieldsDictionaryEncoding()
Expand Down
48 changes: 48 additions & 0 deletions ...src/test/java/org/apache/druid/segment/nested/NestedCommonFormatColumnFormatSpecTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,54 @@ public void testGetEffectiveSpecDefaults()
);
}

@Test
public void testEffectiveSpecIndexSpecOverrides()
{
StringEncodingStrategy frontcoded = new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1);
NestedCommonFormatColumnFormatSpec defaults = NestedCommonFormatColumnFormatSpec.getEffectiveFormatSpec(
null,
IndexSpec.builder()
.withAutoColumnFormatSpec(
NestedCommonFormatColumnFormatSpec.builder()
.setObjectFieldsDictionaryEncoding(frontcoded)
.setObjectStorageEncoding(ObjectStorageEncoding.NONE)
.build()
)
.withMetricCompression(CompressionStrategy.LZF)
.build()
.getEffectiveSpec()
);

Assert.assertEquals(
frontcoded,
defaults.getObjectFieldsDictionaryEncoding()
);
Assert.assertEquals(
ObjectStorageEncoding.NONE,
defaults.getObjectStorageEncoding()
);
Assert.assertEquals(
CompressionStrategy.LZ4,
defaults.getObjectStorageCompression()
);
Assert.assertEquals(
IndexSpec.getDefault().getEffectiveSpec().getDimensionCompression(),
defaults.getDictionaryEncodedColumnCompression()
);
Assert.assertEquals(
IndexSpec.getDefault().getEffectiveSpec().getStringDictionaryEncoding(),
defaults.getStringDictionaryEncoding()
);
Assert.assertEquals(
CompressionStrategy.LZF,
defaults.getLongColumnCompression()
);
Assert.assertEquals(
CompressionStrategy.LZF,
defaults.getDoubleColumnCompression()
);
}

@Test
public void testGetEffectiveSpecMerge()
{
Expand Down