Improve backtesting data memory reuse

CHANGELOG.md

@@ -2,6 +2,21 @@
 
 ## 4.5.66 - Unreleased
 
+### Performance
+- **Backtesting data access avoids retained large timestamp lookup structures and
+  unnecessary bar-frame duplication.** Large `Data.repair_times_and_fill()`
+  frames now rely on the existing nanosecond index search instead of retaining
+  both a pandas `iter_index` Series and a Python datetime lookup dict, and
+  multi-minute `get_bars()` resampling slices the cached OHLCV/corporate-action
+  frame instead of rebuilding a full dataline DataFrame with unused quote
+  columns. Datetime datalines also reuse the existing `DatetimeIndex` backing
+  array instead of allocating a duplicate object array, and PandasData reuses
+  the shared default USD quote asset when normalizing data-store keys.
+- **Order reconciliation now reduces avoidable duplicate work for all brokers.**
+  Strategy order sync indexes LumiBot orders by broker identifier once per
+  reconciliation pass instead of scanning the full order list for each broker
+  order.
+
 ### Fixed
 - **Tradier live polling now throttles repeated broker reads and backs off after
   transient provider 5xx failures.** Account balances, positions, and orders
@@ -14,6 +29,9 @@
   position, and order behavior.** The focused tests cover cache reuse after
   retry-exhausted 5xx responses and ensure OAuth/external-mode polling
   regressions still pass.
+- **Performance regressions now cover large `Data` repair memory, order
+  identifier indexing, and `get_bars()` resampling with non-OHLCV quote
+  columns.**
 
 ## 4.5.65 - Unreleased
 

lumibot/data_sources/pandas_data.py

@@ -69,7 +69,7 @@ def _get_new_pandas_data_key(data):
                 if data.quote is None:
                     # Warn that USD is being used as the quote
                     logger.warning(f"No quote specified for {data.asset}. Using USD as the quote.")
-                    return data.asset, Asset(symbol="USD", asset_type="forex")
+                    return data.asset, _USD_FOREX
                 return data.asset, data.quote
             else:
                 raise ValueError("Asset must be an Asset or a tuple of Asset and quote")

lumibot/strategies/strategy_executor.py

@@ -19,7 +19,6 @@
 from apscheduler.triggers.cron import CronTrigger
 from lumibot.constants import LUMIBOT_DEFAULT_PYTZ
 from lumibot.entities import Asset, Order
-from lumibot.entities import Asset
 from lumibot.strategies.scheduled_timing import ScheduledRunTiming
 from lumibot.tools import append_locals, get_trading_days, staticdecorator
 from lumibot.tools.smart_limit_utils import (
@@ -409,12 +408,13 @@ def sync_broker(self):
         orders_broker = [order for order in orders_broker if order is not None]
         if len(orders_broker) > 0 or self.broker.get_all_orders():
             orders_lumi = self.broker.get_all_orders()
+            orders_lumi_by_identifier = self._index_orders_by_identifier(orders_lumi)
 
             # Check orders at the broker against those in lumibot.
             for order in orders_broker:
                 # Check against existing orders.
-                order_lumi = [ord_lumi for ord_lumi in orders_lumi if ord_lumi.identifier == order.identifier]
-                if len(order_lumi) > 1:
+                order_lumi = orders_lumi_by_identifier.get(order.identifier)
+                if isinstance(order_lumi, list):
                     self.strategy.logger.warning(
                         f"Multiple orders found in lumibot with the same identifier {order.identifier}. "
                         f"This should not happen and indicates a bug in the order tracking. This is manifesting as "
@@ -423,8 +423,6 @@ def sync_broker(self):
                         f"Orders: {order_lumi}"
                     )
                     order_lumi = self.broker._clean_order_trackers(order)
-                else:
-                    order_lumi = order_lumi[0] if len(order_lumi) > 0 else None
 
                 if order_lumi:
                     # Compare the orders.
@@ -563,6 +561,20 @@ def sync_broker(self):
         self.broker._hold_trade_events = False
         self.broker.process_held_trades()
 
+    @staticmethod
+    def _index_orders_by_identifier(orders: list[Order]) -> dict:
+        orders_by_identifier = {}
+        for order in orders:
+            identifier = order.identifier
+            existing = orders_by_identifier.get(identifier)
+            if existing is None:
+                orders_by_identifier[identifier] = order
+            elif isinstance(existing, list):
+                existing.append(order)
+            else:
+                orders_by_identifier[identifier] = [existing, order]
+        return orders_by_identifier
+
     @staticmethod
     def _get_all_order_identifiers(orders_broker: list[Order]) -> set:
         """

tests/test_data_entity.py

@@ -326,6 +326,38 @@ def test_strict_intraday_rejects_multi_minute_history_past_bucket_tolerance(self
         with pytest.raises(ValueError, match="after the available data's end"):
             data.get_bars(base_dt + timedelta(minutes=35), length=3, timestep="5m")
 
+    def test_get_bars_resample_ignores_non_ohlcv_columns(self):
+        asset = Asset("SPY")
+        tz = pytz.timezone("America/New_York")
+        base_dt = tz.localize(datetime(2026, 6, 23, 9, 30))
+        dates = [base_dt + timedelta(minutes=i) for i in range(12)]
+        df = (
+            pd.DataFrame(
+                {
+                    "datetime": dates,
+                    "open": [100.0 + i for i in range(12)],
+                    "high": [101.0 + i for i in range(12)],
+                    "low": [99.0 + i for i in range(12)],
+                    "close": [100.5 + i for i in range(12)],
+                    "volume": [10.0] * 12,
+                    "bid": [100.25 + i for i in range(12)],
+                    "ask": [100.75 + i for i in range(12)],
+                    "last_trade_time": dates,
+                }
+            )
+            .set_index("datetime")
+        )
+        data = Data(asset, df, timestep="minute")
+        expected_data = Data(asset, df[["open", "high", "low", "close", "volume"]].copy(), timestep="minute")
+
+        bars = data.get_bars(base_dt + timedelta(minutes=11), length=2, timestep="5m")
+        expected = expected_data.get_bars(base_dt + timedelta(minutes=11), length=2, timestep="5m")
+
+        assert bars is not None
+        assert expected is not None
+        assert list(bars.columns) == ["open", "high", "low", "close", "volume"]
+        pd.testing.assert_frame_equal(bars, expected)
+
     def test_get_quote_includes_source_bar_provenance(self):
         asset = Asset("BTC", asset_type=Asset.AssetType.CRYPTO)
         tz = pytz.timezone("America/New_York")

tests/test_memory_efficiency_entities_backtesting.py

@@ -0,0 +1,46 @@
+import datetime as dt
+
+import pandas as pd
+
+from lumibot.entities import Asset
+from lumibot.entities.data import Data
+
+
+def _ohlcv_frame(index: pd.DatetimeIndex) -> pd.DataFrame:
+    row_count = len(index)
+    return pd.DataFrame(
+        {
+            "open": range(row_count),
+            "high": range(row_count),
+            "low": range(row_count),
+            "close": range(row_count),
+            "volume": 1,
+        },
+        index=index,
+    )
+
+
+def test_large_data_repair_avoids_retained_iter_index_dict_and_preserves_lookup():
+    index = pd.date_range("2024-01-01", periods=50_001, freq="min", tz="America/New_York")
+    data = Data(Asset("MEM"), _ohlcv_frame(index), timestep="minute")
+
+    data.repair_times_and_fill(index)
+
+    assert data.iter_index_dict == {}
+    assert data.get_iter_count(index[-1].to_pydatetime()) == len(index) - 1
+    assert data.get_iter_count(index[123].to_pydatetime() + dt.timedelta(seconds=30)) == 123
+
+    lazy_iter_index = data.iter_index
+    assert lazy_iter_index.loc[index[123]] == 123
+
+
+def test_data_datetime_dataline_reuses_index_array():
+    index = pd.date_range("2024-01-01", periods=10, freq="min", tz="America/New_York")
+    data = Data(Asset("MEM"), _ohlcv_frame(index), timestep="minute")
+    data.repair_times_and_fill(index)
+
+    datetime_line = data.datalines["datetime"].dataline
+
+    assert datetime_line is data.df.index.array
+    assert data.datetime is datetime_line
+    assert datetime_line[3] == data.df.index[3]

tests/test_pandas_data_find_asset_timestep_match.py

@@ -2,7 +2,7 @@
 
 import pandas as pd
 
-from lumibot.data_sources.pandas_data import PandasData
+from lumibot.data_sources.pandas_data import _USD_FOREX, PandasData
 from lumibot.entities import Asset, Data
 
 
@@ -16,6 +16,17 @@ def _day_df(tz: str = "America/New_York") -> pd.DataFrame:
     return pd.DataFrame({"open": [1, 2, 3], "high": [1, 2, 3], "low": [1, 2, 3], "close": [1, 2, 3], "volume": [0, 0, 0]}, index=idx)
 
 
+def test_set_pandas_data_keys_reuses_default_usd_quote_singleton():
+    base = Asset("AAPL", asset_type=Asset.AssetType.STOCK)
+    data = Data(base, _day_df(), timestep="day")
+
+    keyed = PandasData._set_pandas_data_keys([data])
+    key = next(iter(keyed))
+
+    assert key == (base, Asset("USD", asset_type=Asset.AssetType.FOREX))
+    assert key[1] is _USD_FOREX
+
+
 def test_find_asset_in_data_store_does_not_return_daily_for_minute_requests():
     base = Asset("BTC", asset_type=Asset.AssetType.CRYPTO)
     quote = Asset("USD", asset_type=Asset.AssetType.FOREX)

tests/test_strategy_executor_order_index.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+from lumibot.strategies.strategy_executor import StrategyExecutor
+
+
+def test_index_orders_by_identifier_returns_direct_order_for_unique_ids():
+    first = SimpleNamespace(identifier="1")
+    second = SimpleNamespace(identifier="2")
+
+    indexed = StrategyExecutor._index_orders_by_identifier([first, second])
+
+    assert indexed == {"1": first, "2": second}
+
+
+def test_index_orders_by_identifier_preserves_duplicates_for_cleanup_path():
+    first = SimpleNamespace(identifier="1")
+    duplicate = SimpleNamespace(identifier="1")
+    other = SimpleNamespace(identifier="2")
+
+    indexed = StrategyExecutor._index_orders_by_identifier([first, duplicate, other])
+
+    assert indexed["1"] == [first, duplicate]
+    assert indexed["2"] is other