logdb: range-scan every address-less criterion and cap open-ended spans

api/events_types.go

@@ -233,18 +233,17 @@ func ConvertRange(chain *chain.Chain, r *Range) (*logdb.Range, error) {
 		return &emptyRange, nil
 	}
 
+	// open-ended and oversized ranges end a few blocks above the head
+	to := block.Number(chain.HeadID()) + 6
+	if r.To != nil && *r.To < uint64(to) {
+		to = uint32(*r.To)
+	}
 	from := uint32(0)
 	if r.From != nil {
 		from = uint32(*r.From)
 	}
-
-	to := uint32(logdb.MaxBlockNumber)
-	if r.To != nil && *r.To < logdb.MaxBlockNumber {
-		to = uint32(*r.To)
+	if from > to {
+		return &emptyRange, nil
 	}
-
-	return &logdb.Range{
-		From: from,
-		To:   to,
-	}, nil
+	return &logdb.Range{From: from, To: to}, nil
 }

api/events_types_test.go

@@ -105,8 +105,7 @@ func testConvertRangeWithBlockRangeTypeWithSwitchedFromAndTo(t *testing.T, chain
 	convertedRng, err := ConvertRange(chain.Repo().NewBestChain(), rng)
 
 	assert.NoError(t, err)
-	assert.Equal(t, emptyRange.From, convertedRng.From)
-	assert.Equal(t, uint32(*rng.To), convertedRng.To)
+	assert.Equal(t, &emptyRange, convertedRng)
 }
 
 func testConvertRangeWithTimeRangeTypeLessThenGenesis(t *testing.T, chain *testchain.Chain) {
@@ -169,6 +168,7 @@ func TestConvertRange_Matrix(t *testing.T) {
 	bestChain := chain.Repo().NewBestChain()
 	genesis := chain.GenesisBlock().Header()
 	bestBlock := chain.Repo().BestBlockSummary().Header
+	head := bestBlock.Number()
 
 	// Cases here focus on what TestEventsTypes (which builds ranges via newRange
 	// with non-nil fields) does not cover: the JSON path, omitted-field defaulting,
@@ -186,7 +186,7 @@ func TestConvertRange_Matrix(t *testing.T) {
 		{
 			name:     "empty object returns full block range",
 			json:     `{}`,
-			expected: &logdb.Range{From: 0, To: uint32(logdb.MaxBlockNumber)},
+			expected: &logdb.Range{From: 0, To: head + 6},
 		},
 
 		// Block range - omitted-field defaulting and boundary capping.
@@ -196,19 +196,34 @@ func TestConvertRange_Matrix(t *testing.T) {
 			expected: &logdb.Range{From: 0, To: 10},
 		},
 		{
-			name:     "block range with omitted to defaults to MaxBlockNumber",
+			name:     "block range with omitted to defaults to head+6",
 			json:     `{"unit": "block", "from": 5}`,
-			expected: &logdb.Range{From: 5, To: uint32(logdb.MaxBlockNumber)},
+			expected: &logdb.Range{From: 5, To: head + 6},
 		},
 		{
 			name:     "block range at MaxBlockNumber",
 			json:     `{"unit": "block", "from": 268435455, "to": 268435455}`,
-			expected: &logdb.Range{From: uint32(logdb.MaxBlockNumber), To: uint32(logdb.MaxBlockNumber)},
+			expected: &emptyRange,
 		},
 		{
 			name:     "block range to exceeds MaxBlockNumber is capped",
 			json:     `{"unit": "block", "from": 5, "to": 999999999999}`,
-			expected: &logdb.Range{From: 5, To: uint32(logdb.MaxBlockNumber)},
+			expected: &logdb.Range{From: 5, To: head + 6},
+		},
+		{
+			name:     "block range to above head+6 is capped",
+			json:     fmt.Sprintf(`{"unit": "block", "from": 0, "to": %d}`, head+7),
+			expected: &logdb.Range{From: 0, To: head + 6},
+		},
+		{
+			name:     "block range from above head+6 is empty",
+			json:     fmt.Sprintf(`{"unit": "block", "from": %d}`, head+7),
+			expected: &emptyRange,
+		},
+		{
+			name:     "block range to below head is kept",
+			json:     `{"unit": "block", "from": 0, "to": 0}`,
+			expected: &logdb.Range{From: 0, To: 0},
 		},
 
 		// Time range - omitted-field defaulting.

logdb/logdb.go

@@ -230,8 +230,14 @@ func (db *LogDB) Path() string {
 	return db.path
 }
 
-// buildEventFilterSubQuery builds the seq-selecting subquery for a non-nil filter.
-func buildEventFilterSubQuery(filter *EventFilter) (string, []any, error) {
+// narrowRange reports whether r spans fewer than maxScanBlocks blocks.
+func narrowRange(r *Range) bool {
+	return r != nil && r.To >= r.From && r.To-r.From < maxScanBlocks
+}
+
+// buildEventFilterSubQuery builds the seq-selecting subquery for a non-nil filter. With narrow set,
+// criteria whose index lookup the seq range cannot bound are evaluated by a seq range scan instead.
+func buildEventFilterSubQuery(filter *EventFilter, narrow bool) (string, []any, error) {
 	var (
 		subQuery = "SELECT seq FROM event WHERE 1"
 		args     []any
@@ -256,10 +262,8 @@ func buildEventFilterSubQuery(filter *EventFilter) (string, []any, error) {
 	if len(filter.CriteriaSet) > 0 {
 		var sb strings.Builder
 		sb.WriteString(" AND (")
-
-		narrow := filter.Range != nil && filter.Range.To-filter.Range.From < maxScanBlocks
 		for i, c := range filter.CriteriaSet {
-			cond, cargs := c.toWhereCondition(narrow && c.topic0Only())
+			cond, cargs := c.toWhereCondition(narrow && !c.seqBounded())
 			if i > 0 {
 				sb.WriteString(" OR")
 			}
@@ -305,7 +309,7 @@ FROM (%v) e
 
 	metricsHandleEventsFilter(filter)
 
-	subQuery, args, err := buildEventFilterSubQuery(filter)
+	subQuery, args, err := buildEventFilterSubQuery(filter, narrowRange(filter.Range))
 	if err != nil {
 		return nil, err
 	}

logdb/plan_test.go

@@ -27,7 +27,7 @@ func TestEventQueryPlanPerCriteriaShape(t *testing.T) {
 	defer db.Close()
 
 	addr := randAddress()
-	t0, t1, t2, t3 := randBytes32(), randBytes32(), randBytes32(), randBytes32()
+	t0, t1, t2, t3, t4 := randBytes32(), randBytes32(), randBytes32(), randBytes32(), randBytes32()
 	crit := func(a *thor.Address, topics ...*thor.Bytes32) *EventCriteria {
 		c := &EventCriteria{Address: a}
 		copy(c.Topics[:], topics)
@@ -52,15 +52,17 @@ func TestEventQueryPlanPerCriteriaShape(t *testing.T) {
 		{"six topic0", six, expect{scanPlan, "MULTI-INDEX OR", "MULTI-INDEX OR"}},
 		{"address", []*EventCriteria{crit(&addr)}, expect{"INDEX event_i0 ", "INDEX event_i0 ", "INDEX event_i0 "}},
 		{"address+topic0", []*EventCriteria{crit(&addr, &t0)}, expect{"INDEX event_i1 ", "INDEX event_i1 ", "INDEX event_i1 "}},
-		{"topic0+topic1", []*EventCriteria{crit(nil, &t0, &t1)}, expect{"INDEX event_i2 ", "INDEX event_i2 ", "INDEX event_i2 "}},
-		{"topic0+topic2", []*EventCriteria{crit(nil, &t0, nil, &t2)}, expect{"INDEX event_i3 ", "INDEX event_i3 ", "INDEX event_i3 "}},
-		{"topic0+topic3", []*EventCriteria{crit(nil, &t0, nil, nil, &t3)}, expect{"INDEX event_i4 ", "INDEX event_i4 ", "INDEX event_i4 "}},
+		{"topic1", []*EventCriteria{crit(nil, nil, &t1)}, expect{scanPlan, "INDEX event_i2 ", "INDEX event_i2 "}},
+		{"topic4", []*EventCriteria{crit(nil, nil, nil, nil, nil, &t4)}, expect{scanPlan, "INDEX event_i5 ", "INDEX event_i5 "}},
+		{"topic0+topic1", []*EventCriteria{crit(nil, &t0, &t1)}, expect{scanPlan, "INDEX event_i2 ", "INDEX event_i2 "}},
+		{"topic0+topic2", []*EventCriteria{crit(nil, &t0, nil, &t2)}, expect{scanPlan, "INDEX event_i3 ", "INDEX event_i3 "}},
+		{"topic0+topic3", []*EventCriteria{crit(nil, &t0, nil, nil, &t3)}, expect{scanPlan, "INDEX event_i4 ", "INDEX event_i4 "}},
 		{"address+topic0+topic1", []*EventCriteria{crit(&addr, &t0, &t1)}, expect{"INDEX event_i2 ", "INDEX event_i2 ", "INDEX event_i2 "}},
 		{"mixed", []*EventCriteria{crit(nil, &t0), crit(&addr, &t0)}, expect{scanPlan, "MULTI-INDEX OR", "MULTI-INDEX OR"}},
 	}
 
 	plan := func(t *testing.T, set []*EventCriteria, r *Range) string {
-		sub, args, err := buildEventFilterSubQuery(&EventFilter{CriteriaSet: set, Range: r})
+		sub, args, err := buildEventFilterSubQuery(&EventFilter{CriteriaSet: set, Range: r}, narrowRange(r))
 		require.NoError(t, err)
 		rows, err := db.readDB.Query("EXPLAIN QUERY PLAN "+sub, args...)
 		require.NoError(t, err)
@@ -102,12 +104,29 @@ func TestEventQueryPlanPerCriteriaShape(t *testing.T) {
 	})
 }
 
+func TestNarrowRange(t *testing.T) {
+	for _, c := range []struct {
+		name string
+		rng  *Range
+		want bool
+	}{
+		{"nil", nil, false},
+		{"inverted", &Range{From: 10, To: 5}, false},
+		{"narrow", &Range{From: 0, To: maxScanBlocks - 1}, true},
+		{"wide", &Range{From: 0, To: maxScanBlocks}, false},
+		{"single block", &Range{From: 5, To: 5}, true},
+	} {
+		t.Run(c.name, func(t *testing.T) { assert.Equal(t, c.want, narrowRange(c.rng)) })
+	}
+}
+
 func TestEventFilterScanEquivalence(t *testing.T) {
 	db, err := NewMem()
 	require.NoError(t, err)
 	defer db.Close()
 
 	topics := []thor.Bytes32{randBytes32(), randBytes32(), randBytes32()}
+	topics1 := []thor.Bytes32{randBytes32(), randBytes32()}
 	addrs := []thor.Address{randAddress(), randAddress()}
 
 	b := new(block.Builder).Build()
@@ -118,8 +137,8 @@ func TestEventFilterScanEquivalence(t *testing.T) {
 			bb.Transaction(newTx(tx.TypeLegacy))
 			k := n*4 + j
 			receipts = append(receipts, &tx.Receipt{Outputs: []*tx.Output{{Events: tx.Events{
-				{Address: addrs[k%2], Topics: []thor.Bytes32{topics[k%3]}, Data: []byte{1}},
-				{Address: addrs[(k+1)%2], Topics: []thor.Bytes32{topics[(k+1)%3]}, Data: []byte{2}},
+				{Address: addrs[k%2], Topics: []thor.Bytes32{topics[k%3], topics1[k%2]}, Data: []byte{1}},
+				{Address: addrs[(k+1)%2], Topics: []thor.Bytes32{topics[(k+1)%3], topics1[(k+1)%2]}, Data: []byte{2}},
 			}}}})
 		}
 		b = bb.Build()
@@ -129,35 +148,17 @@ func TestEventFilterScanEquivalence(t *testing.T) {
 	}
 
 	type key [3]uint32
-	crit := func(i int) *EventCriteria { return &EventCriteria{Topics: [5]*thor.Bytes32{&topics[i]}} }
+	crit := func(a *thor.Address, ts ...*thor.Bytes32) *EventCriteria {
+		c := &EventCriteria{Address: a}
+		copy(c.Topics[:], ts)
+		return c
+	}
 
-	// index path: same subquery shape, criteria rendered without the unary plus
+	// index path: the same subquery with the range scan disabled
 	indexPath := func(f *EventFilter) []key {
-		var (
-			sub  = "SELECT seq FROM event WHERE 1"
-			args []any
-		)
-		from, _ := newSequence(f.Range.From, 0, 0)
-		to, _ := newSequence(f.Range.To, txIndexMask, logIndexMask)
-		args = append(args, from, to)
-		var sb strings.Builder
-		sb.WriteString(" AND seq >= ? AND seq <= ? AND (")
-		for i, c := range f.CriteriaSet {
-			cond, cargs := c.toWhereCondition(false)
-			if i > 0 {
-				sb.WriteString(" OR")
-			}
-			sb.WriteString(" (" + cond + ")")
-			args = append(args, cargs...)
-		}
-		sb.WriteString(")")
-		sub += sb.String() //nolint:gosec // test query from fixed fragments
-		if f.Order == DESC {
-			sub += " ORDER BY seq DESC LIMIT ?, ?"
-		} else {
-			sub += " ORDER BY seq ASC LIMIT ?, ?"
-		}
-		args = append(args, f.Options.Offset, f.Options.Limit)
+		sub, args, err := buildEventFilterSubQuery(f, false)
+		require.NoError(t, err)
+		require.NotContains(t, sub, "+topic")
 		rows, err := db.readDB.Query(sub, args...)
 		require.NoError(t, err)
 		defer rows.Close()
@@ -171,26 +172,33 @@ func TestEventFilterScanEquivalence(t *testing.T) {
 		return out
 	}
 
-	for _, set := range [][]*EventCriteria{{crit(0)}, {crit(1), crit(2)}, {crit(0), crit(1), crit(2)}} {
+	sets := map[string][]*EventCriteria{
+		"topic0":        {crit(nil, &topics[0])},
+		"two topic0":    {crit(nil, &topics[1]), crit(nil, &topics[2])},
+		"three topic0":  {crit(nil, &topics[0]), crit(nil, &topics[1]), crit(nil, &topics[2])},
+		"topic0+topic1": {crit(nil, &topics[0], &topics1[0])},
+		"topic1":        {crit(nil, nil, &topics1[1])},
+		"mixed":         {crit(nil, &topics[0]), crit(&addrs[0], &topics[1])},
+	}
+	for name, set := range sets {
 		for _, order := range []Order{ASC, DESC} {
 			for _, rng := range []Range{{5, 5}, {3, 12}, {0, 100}} {
 				for _, opt := range []Options{{0, 1000}, {2, 5}, {1000, 5}} {
 					f := &EventFilter{CriteriaSet: set, Range: &rng, Options: &opt, Order: order}
-					name := fmt.Sprintf("%d/%v/%v/%v", len(set), order, rng, opt)
-					t.Run(name, func(t *testing.T) {
-						sub, _, err := buildEventFilterSubQuery(f)
+					t.Run(fmt.Sprintf("%s/%v/%v/%v", name, order, rng, opt), func(t *testing.T) {
+						require.True(t, narrowRange(f.Range))
+						sub, _, err := buildEventFilterSubQuery(f, true)
 						require.NoError(t, err)
-						require.Contains(t, sub, "+topic0")
+						require.Contains(t, sub, "+topic")
 
 						evs, err := db.FilterEvents(context.Background(), f)
 						require.NoError(t, err)
 						var got []key
 						for _, e := range evs {
 							got = append(got, key{e.BlockNumber, e.TxIndex, e.LogIndex})
 						}
-						want := indexPath(f)
-						assert.Equal(t, want, got)
-						if rng.To-rng.From == 0 && opt.Offset == 0 && opt.Limit == 1000 {
+						assert.Equal(t, indexPath(f), got)
+						if rng == (Range{0, 100}) && opt == (Options{0, 1000}) {
 							assert.NotEmpty(t, got)
 						}
 					})

logdb/types.go

@@ -64,38 +64,32 @@ type EventCriteria struct {
 	Topics  [5]*thor.Bytes32
 }
 
-// maxScanBlocks bounds the block span over which topic0-only criteria are evaluated by a seq
-// range scan rather than the topic0 index; past this width the scan stops paying off for sparse topics.
+// maxScanBlocks bounds the block span over which address-less criteria are evaluated by a seq
+// range scan rather than a topic index; past this width the scan stops paying off for sparse topics.
 const maxScanBlocks = 100_000
 
-// topic0Only reports whether the criterion constrains nothing but topic0.
-func (c *EventCriteria) topic0Only() bool {
-	if c.Address != nil || c.Topics[0] == nil {
-		return false
-	}
-	for _, t := range c.Topics[1:] {
-		if t != nil {
-			return false
-		}
-	}
-	return true
+// seqBounded reports whether an index lookup for the criterion can be bounded by the seq range.
+// Every event index except event_i0 has address between its topic columns and seq, so without an
+// address the lookup reads every matching entry chain-wide before the seq filter applies.
+func (c *EventCriteria) seqBounded() bool {
+	return c.Address != nil
 }
 
-// toWhereCondition renders the criterion. With noTopic0Index set, topic0 is written as +topic0 so
-// SQLite cannot use it for an index lookup and falls back to the seq range.
-func (c *EventCriteria) toWhereCondition(noTopic0Index bool) (cond string, args []any) {
+// toWhereCondition renders the criterion. With rangeScan set, topics are written as +topicN so
+// SQLite cannot use them for an index lookup and falls back to the seq range.
+func (c *EventCriteria) toWhereCondition(rangeScan bool) (cond string, args []any) {
 	cond = "1"
 	if c.Address != nil {
 		cond += " AND address = " + refIDQuery
 		args = append(args, c.Address.Bytes())
 	}
 	for i, topic := range c.Topics {
 		if topic != nil {
-			if i == 0 && noTopic0Index {
-				cond += " AND +topic0 = " + refIDQuery
-			} else {
-				cond += fmt.Sprintf(" AND topic%v = ", i) + refIDQuery
+			col := fmt.Sprintf("topic%v", i)
+			if rangeScan {
+				col = "+" + col
 			}
+			cond += " AND " + col + " = " + refIDQuery
 			args = append(args, removeLeadingZeros(topic.Bytes()))
 		}
 	}