Skip to content

Include TSDB head max time in out of bounds and too old sample errors #7695

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yeya24 wants to merge 3 commits into
base: master
Choose a base branch
from
+58 −5
Open

Include TSDB head max time in out of bounds and too old sample errors #7695

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
796bbf1
Include TSDB head max time in out of bounds and too old sample errors
yeya24 Jul 17, 2026
06381f9
Add CHANGELOG entry
yeya24 Jul 17, 2026
27881e2
Snapshot head max time at appender creation instead of per-error lookup
yeya24 Jul 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
* [ENHANCEMENT] Query Frontend: Rename `time_taken` field to `time_taken_ms` and make it return millisecond count. #7649
* [ENHANCEMENT] Update prometheus alertmanager version to v0.33.0. #7647
* [ENHANCEMENT] Querier/Ingester: Detach ingester series from gRPC buffers to reduce heap. #7670
* [ENHANCEMENT] Ingester: Include the TSDB head max time in the `out of bounds` and `too old sample` error messages, so that users can see how far behind the accepted time range a rejected sample is. #7695
* [BUGFIX] Querier: Fix queryWithRetry and labelsWithRetry returning (nil, nil) on cancelled context by propagating ctx.Err(). #7370
* [BUGFIX] Metrics Helper: Fix non-deterministic bucket order in merged histograms by sorting buckets after map iteration, matching Prometheus client library behavior. #7380
* [BUGFIX] Distributor: Return HTTP 401 Unauthorized when tenant ID resolution fails in the Prometheus Remote Write 2.0 path. #7389
Expand Down
34 changes: 32 additions & 2 deletions pkg/ingester/ingester.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ const (
errTSDBCreateIncompatibleState = "cannot create a new TSDB while the ingester is not in active state (current state: %s)"
errTSDBIngestWithTimestamp = "err: %v. series=%s" // Using error.Wrap puts the message before the error and if the series is too long, its truncated.
errTSDBIngest = "err: %v. timestamp=%s, series=%s" // Using error.Wrap puts the message before the error and if the series is too long, its truncated.
errTSDBIngestWithHeadMaxTime = "err: %v. timestamp=%s, tsdbHeadMaxTimestamp=%s, series=%s"
errTSDBIngestExemplar = "err: %v. timestamp=%s, series=%s, exemplar=%s"

// Jitter applied to the idle timeout to prevent compaction in all ingesters concurrently.
Expand Down Expand Up @@ -1417,6 +1418,13 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte
perMetricSeriesLimitCount = 0
discardedNativeHistogramCount = 0

// headMaxTime is snapshotted right after the appender is created. The appender
// derives its lower time bound checks (e.g. out of bounds) from the head max
// time at creation time. We cannot access the appender's internal bound
// directly, so we snapshot the head max time once here instead of fetching it
// again for every failed sample.
headMaxTime int64

updateFirstPartial = func(errFn func() error) {
if firstPartialErr == nil {
firstPartialErr = errFn()
Expand All @@ -1432,7 +1440,9 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte
case errors.Is(cause, storage.ErrOutOfBounds):
sampleOutOfBoundsCount++
i.validateMetrics.DiscardedSeriesTracker.Track(sampleOutOfBounds, userID, copiedLabels.Hash())
updateFirstPartial(func() error { return wrappedTSDBIngestErr(err, model.Time(timestampMs), lbls) })
updateFirstPartial(func() error {
return wrappedTSDBIngestErrWithHeadMaxTime(err, model.Time(timestampMs), model.Time(headMaxTime), lbls)
})

case errors.Is(cause, storage.ErrOutOfOrderSample):
sampleOutOfOrderCount++
Expand All @@ -1447,7 +1457,9 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte
case errors.Is(cause, storage.ErrTooOldSample):
sampleTooOldCount++
i.validateMetrics.DiscardedSeriesTracker.Track(sampleTooOld, userID, copiedLabels.Hash())
updateFirstPartial(func() error { return wrappedTSDBIngestErr(err, model.Time(timestampMs), lbls) })
updateFirstPartial(func() error {
return wrappedTSDBIngestErrWithHeadMaxTime(err, model.Time(timestampMs), model.Time(headMaxTime), lbls)
})

case errors.Is(cause, errMaxSeriesPerUserLimitExceeded):
perUserSeriesLimitCount++
Expand Down Expand Up @@ -1493,6 +1505,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte

// Walk the samples, appending them to the users database
app := db.Appender(ctx).(extendedAppender)
headMaxTime = db.Head().MaxTime()

// Ensure the appender is always released so that we don't leak TSDB head
// series references, mmap'd chunks and pending state on early returns.
Expand Down Expand Up @@ -3947,6 +3960,23 @@ func wrappedTSDBIngestErr(ingestErr error, timestamp model.Time, labels []cortex
}
}

// wrappedTSDBIngestErrWithHeadMaxTime is like wrappedTSDBIngestErr, but it also includes the
// TSDB head max time in the error message. This helps users understand how far behind the
// accepted time range a rejected sample is, e.g. for out of bounds or too old sample errors.
func wrappedTSDBIngestErrWithHeadMaxTime(ingestErr error, timestamp, headMaxTime model.Time, labels []cortexpb.LabelAdapter) error {
if ingestErr == nil {
return nil
}

// The TSDB head max time is unset when the head is empty (e.g. no sample ingested yet
// after startup). Fall back to the error message without the head max time in that case.
if int64(headMaxTime) == math.MinInt64 {
return wrappedTSDBIngestErr(ingestErr, timestamp, labels)
}

return fmt.Errorf(errTSDBIngestWithHeadMaxTime, ingestErr, timestamp.Time().UTC().Format(time.RFC3339Nano), headMaxTime.Time().UTC().Format(time.RFC3339Nano), cortexpb.FromLabelAdaptersToLabels(labels).String())
}

func wrappedTSDBIngestExemplarErr(ingestErr error, timestamp model.Time, seriesLabels, exemplarLabels []cortexpb.LabelAdapter) error {
if ingestErr == nil {
return nil
Expand Down
28 changes: 25 additions & 3 deletions pkg/ingester/ingester_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1504,7 +1504,7 @@ func TestIngester_Push(t *testing.T) {
},
cortexpb.API),
},
expectedErr: httpgrpc.Errorf(http.StatusBadRequest, "%s", wrapWithUser(wrappedTSDBIngestErr(storage.ErrOutOfBounds, model.Time(1575043969-(86400*1000)), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()),
expectedErr: httpgrpc.Errorf(http.StatusBadRequest, "%s", wrapWithUser(wrappedTSDBIngestErrWithHeadMaxTime(storage.ErrOutOfBounds, model.Time(1575043969-(86400*1000)), model.Time(1575043969), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()),
expectedIngested: []cortexpb.TimeSeries{
{Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 1575043969}}},
},
Expand Down Expand Up @@ -1561,7 +1561,7 @@ func TestIngester_Push(t *testing.T) {
cortexpb.API),
},
oooTimeWindow: 5 * time.Minute,
expectedErr: httpgrpc.Errorf(http.StatusBadRequest, "%s", wrapWithUser(wrappedTSDBIngestErr(storage.ErrTooOldSample, model.Time(1575043969-(600*1000)), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()),
expectedErr: httpgrpc.Errorf(http.StatusBadRequest, "%s", wrapWithUser(wrappedTSDBIngestErrWithHeadMaxTime(storage.ErrTooOldSample, model.Time(1575043969-(600*1000)), model.Time(1575043969), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()),
expectedIngested: []cortexpb.TimeSeries{
{Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 1575043969}}},
},
Expand Down Expand Up @@ -1663,7 +1663,7 @@ func TestIngester_Push(t *testing.T) {
cortexpb.API),
},
oooTimeWindow: 5 * time.Minute,
expectedErr: httpgrpc.Errorf(http.StatusBadRequest, "%s", wrapWithUser(wrappedTSDBIngestErr(storage.ErrTooOldSample, model.Time(1575043969-(600*1000)), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()),
expectedErr: httpgrpc.Errorf(http.StatusBadRequest, "%s", wrapWithUser(wrappedTSDBIngestErrWithHeadMaxTime(storage.ErrTooOldSample, model.Time(1575043969-(600*1000)), model.Time(1575043969), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()),
expectedIngested: []cortexpb.TimeSeries{
{Labels: metricLabelAdapters, Histograms: []cortexpb.WrappedHistogram{cortexpb.WrapHistogram(cortexpb.HistogramToHistogramProto(1575043969, tsdbutil.GenerateTestHistogram(1)))}},
},
Expand Down Expand Up @@ -8359,3 +8359,25 @@ func TestIngester_DiscardOutOfOrderFlagIntegration(t *testing.T) {
require.NoError(t, iter.Err())
require.Equal(t, 1, sampleCount, "Should have exactly one sample stored")
}

func TestWrappedTSDBIngestErrWithHeadMaxTime(t *testing.T) {
lbls := cortexpb.FromLabelsToLabelAdapters(labels.FromStrings(labels.MetricName, "test"))

// The error message should include both the sample timestamp and the TSDB head max time.
err := wrappedTSDBIngestErrWithHeadMaxTime(storage.ErrOutOfBounds, model.Time(1575043969), model.Time(1575043969+60000), lbls)
require.Error(t, err)
require.Equal(t, `err: out of bounds. timestamp=1970-01-19T05:30:43.969Z, tsdbHeadMaxTimestamp=1970-01-19T05:31:43.969Z, series={__name__="test"}`, err.Error())

err = wrappedTSDBIngestErrWithHeadMaxTime(storage.ErrTooOldSample, model.Time(1575043969), model.Time(1575043969+60000), lbls)
require.Error(t, err)
require.Equal(t, `err: too old sample. timestamp=1970-01-19T05:30:43.969Z, tsdbHeadMaxTimestamp=1970-01-19T05:31:43.969Z, series={__name__="test"}`, err.Error())

// When the TSDB head max time is unset (empty head), fall back to the error message
// without the head max time.
err = wrappedTSDBIngestErrWithHeadMaxTime(storage.ErrOutOfBounds, model.Time(1575043969), model.Time(math.MinInt64), lbls)
require.Error(t, err)
require.Equal(t, `err: out of bounds. timestamp=1970-01-19T05:30:43.969Z, series={__name__="test"}`, err.Error())

// A nil ingest error returns nil.
require.NoError(t, wrappedTSDBIngestErrWithHeadMaxTime(nil, model.Time(1575043969), model.Time(1575043969), lbls))
}