Skip to content

Support ingesting NHCB #7191

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yeya24 merged 1 commit into from
Jan 8, 2026
Merged

Support ingesting NHCB #7191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
* [FEATURE] Distributor: Add a per-tenant flag `-distributor.enable-type-and-unit-labels` that enables adding `__unit__` and `__type__` labels for remote write v2 and OTLP requests. This is a breaking change; the `-distributor.otlp.enable-type-and-unit-labels` flag is now deprecated, operates as a no-op, and has been consolidated into this new flag. #7077
* [FEATURE] Querier: Add experimental projection pushdown support in Parquet Queryable. #7152
* [FEATURE] Ingester: Add experimental active series queried metric. #7173
* [ENHANCEMENT] Ingester: Add support for ingesting Native Histogram with Custom Buckets. #7191
* [ENHANCEMENT] Ingester: Optimize labels out-of-order (ooo) check by allowing the iteration to terminate immediately upon finding the first unsorted label. #7186
* [ENHANCEMENT] Distributor: Skip attaching `__unit__` and `__type__` labels when `-distributor.enable-type-and-unit-labels` is enabled, as these are appended from metadata. #7145
* [ENHANCEMENT] StoreGateway: Add tracings to parquet mode. #7125
Expand Down
28 changes: 22 additions & 6 deletions integration/e2e/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,12 +292,19 @@ func CreateNHBlock(
var ref storage.SeriesRef
start := RandRange(rnd, mint, maxt)
for j := range numNHSamples {
if num%2 == 0 {
switch num % 4 {
case 0:
// append float histogram
ref, err = app.AppendHistogram(ref, series[i], start, nil, tsdbutil.GenerateTestFloatHistogram(int64(i+j)))
} else {
// append histogram
case 1:
// append int histogram
ref, err = app.AppendHistogram(ref, series[i], start, tsdbutil.GenerateTestHistogram(int64(i+j)), nil)
case 2:
// append float histogram with custom bucket
ref, err = app.AppendHistogram(ref, series[i], start, nil, tsdbutil.GenerateTestCustomBucketsFloatHistogram(int64(i+j)))
case 3:
// append int histogram with custom bucket
ref, err = app.AppendHistogram(ref, series[i], start, tsdbutil.GenerateTestCustomBucketsHistogram(int64(i+j)), nil)
}
if err != nil {
if rerr := app.Rollback(); rerr != nil {
Expand Down Expand Up @@ -424,7 +431,7 @@ func CreateBlock(
return id, nil
}

func GenerateHistogramSeriesV2(name string, ts time.Time, i uint32, floatHistogram bool, additionalLabels ...prompb.Label) (symbols []string, series []writev2.TimeSeries) {
func GenerateHistogramSeriesV2(name string, ts time.Time, i uint32, customBucket, floatHistogram bool, additionalLabels ...prompb.Label) (symbols []string, series []writev2.TimeSeries) {
tsMillis := TimeToMilliseconds(ts)

st := writev2.NewSymbolTable()
Expand All @@ -439,11 +446,20 @@ func GenerateHistogramSeriesV2(name string, ts time.Time, i uint32, floatHistogr
fh *histogram.FloatHistogram
ph writev2.Histogram
)

if floatHistogram {
fh = tsdbutil.GenerateTestFloatHistogram(int64(i))
if customBucket {
fh = tsdbutil.GenerateTestCustomBucketsFloatHistogram(int64(i))
} else {
fh = tsdbutil.GenerateTestFloatHistogram(int64(i))
}
ph = writev2.FromFloatHistogram(tsMillis, fh)
} else {
h = tsdbutil.GenerateTestHistogram(int64(i))
if customBucket {
h = tsdbutil.GenerateTestCustomBucketsHistogram(int64(i))
} else {
h = tsdbutil.GenerateTestHistogram(int64(i))
}
ph = writev2.FromIntHistogram(tsMillis, h)
}

Expand Down
44 changes: 34 additions & 10 deletions integration/remote_write_v2_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,12 @@ func TestIngesterRollingUpdate(t *testing.T) {

// histogram
histogramIdx := rand.Uint32()
symbols2, histogramSeries := e2e.GenerateHistogramSeriesV2("test_histogram", now, histogramIdx, false, prompb.Label{Name: "job", Value: "test"}, prompb.Label{Name: "float", Value: "false"})
symbols2, histogramSeries := e2e.GenerateHistogramSeriesV2("test_histogram", now, histogramIdx, false, false, prompb.Label{Name: "job", Value: "test"}, prompb.Label{Name: "float", Value: "false"})
writeStats, err := c.PushV2(symbols2, histogramSeries)
require.NoError(t, err)
testPushHeader(t, writeStats, 0, 1, 0)

symbols3, histogramFloatSeries := e2e.GenerateHistogramSeriesV2("test_histogram", now, histogramIdx, true, prompb.Label{Name: "job", Value: "test"}, prompb.Label{Name: "float", Value: "true"})
symbols3, histogramFloatSeries := e2e.GenerateHistogramSeriesV2("test_histogram", now, histogramIdx, false, true, prompb.Label{Name: "job", Value: "test"}, prompb.Label{Name: "float", Value: "true"})
writeStats, err = c.PushV2(symbols3, histogramFloatSeries)
require.NoError(t, err)
testPushHeader(t, writeStats, 0, 1, 0)
Expand Down Expand Up @@ -349,28 +349,52 @@ func TestIngest(t *testing.T) {

// histogram
histogramIdx := rand.Uint32()
symbols2, histogramSeries := e2e.GenerateHistogramSeriesV2("test_histogram", now, histogramIdx, false, prompb.Label{Name: "job", Value: "test"}, prompb.Label{Name: "float", Value: "false"})
writeStats, err = c.PushV2(symbols2, histogramSeries)
symbols2, intNH := e2e.GenerateHistogramSeriesV2("test_nh", now, histogramIdx, false, false, prompb.Label{Name: "job", Value: "test"}, prompb.Label{Name: "float", Value: "false"})
writeStats, err = c.PushV2(symbols2, intNH)
require.NoError(t, err)
testPushHeader(t, writeStats, 0, 1, 0)

// float histogram
symbols3, histogramFloatSeries := e2e.GenerateHistogramSeriesV2("test_histogram", now, histogramIdx, true, prompb.Label{Name: "job", Value: "test"}, prompb.Label{Name: "float", Value: "true"})
writeStats, err = c.PushV2(symbols3, histogramFloatSeries)
symbols3, floatNH := e2e.GenerateHistogramSeriesV2("test_nh", now, histogramIdx, false, true, prompb.Label{Name: "job", Value: "test"}, prompb.Label{Name: "float", Value: "true"})
writeStats, err = c.PushV2(symbols3, floatNH)
require.NoError(t, err)
testPushHeader(t, writeStats, 0, 1, 0)

// histogram with Custom Bucket
symbols4, intNHCB := e2e.GenerateHistogramSeriesV2("test_nhcb", now, histogramIdx, true, false, prompb.Label{Name: "job", Value: "test"}, prompb.Label{Name: "float", Value: "false"})
writeStats, err = c.PushV2(symbols4, intNHCB)
require.NoError(t, err)
testPushHeader(t, writeStats, 0, 1, 0)

// float histogram with Custom Bucket
symbols5, floatNHCB := e2e.GenerateHistogramSeriesV2("test_nhcb", now, histogramIdx, true, true, prompb.Label{Name: "job", Value: "test"}, prompb.Label{Name: "float", Value: "true"})
writeStats, err = c.PushV2(symbols5, floatNHCB)
require.NoError(t, err)
testPushHeader(t, writeStats, 0, 1, 0)

testHistogramTimestamp := now.Add(blockRangePeriod * 2)
expectedHistogram := tsdbutil.GenerateTestHistogram(int64(histogramIdx))
result, err = c.Query(`test_histogram`, testHistogramTimestamp)
expectedNH := tsdbutil.GenerateTestHistogram(int64(histogramIdx))
result, err = c.Query(`test_nh`, testHistogramTimestamp)
require.NoError(t, err)
require.Equal(t, model.ValVector, result.Type())
v := result.(model.Vector)
require.Equal(t, 2, v.Len())
for _, s := range v {
require.NotNil(t, s.Histogram)
require.Equal(t, float64(expectedHistogram.Count), float64(s.Histogram.Count))
require.Equal(t, float64(expectedHistogram.Sum), float64(s.Histogram.Sum))
require.Equal(t, float64(expectedNH.Count), float64(s.Histogram.Count))
require.Equal(t, float64(expectedNH.Sum), float64(s.Histogram.Sum))
}

expectedNHCB := tsdbutil.GenerateTestCustomBucketsHistogram(int64(histogramIdx))
result, err = c.Query(`test_nhcb`, testHistogramTimestamp)
require.NoError(t, err)
require.Equal(t, model.ValVector, result.Type())
v = result.(model.Vector)
require.Equal(t, 2, v.Len())
for _, s := range v {
require.NotNil(t, s.Histogram)
require.Equal(t, float64(expectedNHCB.Count), float64(s.Histogram.Count))
require.Equal(t, float64(expectedNHCB.Sum), float64(s.Histogram.Sum))
}
}

Expand Down
Loading
Loading