From f02dda0e3b644498da501fc6583a1cd1a14cf336 Mon Sep 17 00:00:00 2001 From: Al4ise Date: Sat, 4 Jul 2026 19:24:01 +0300 Subject: [PATCH 1/3] Improve core backtesting data efficiency --- CHANGELOG.md | 16 ++ lumibot/entities/data.py | 249 +++++++++--------- lumibot/strategies/strategy_executor.py | 22 +- tests/test_data_entity.py | 32 +++ ..._memory_efficiency_entities_backtesting.py | 34 +++ tests/test_strategy_executor_order_index.py | 25 ++ 6 files changed, 251 insertions(+), 127 deletions(-) create mode 100644 tests/test_memory_efficiency_entities_backtesting.py create mode 100644 tests/test_strategy_executor_order_index.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d29e7a646..771ae668f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ ## 4.5.66 - Unreleased +### Performance +- **Backtesting data access avoids retained large timestamp lookup structures and + unnecessary bar-frame duplication.** Large `Data.repair_times_and_fill()` + frames now rely on the existing nanosecond index search instead of retaining + both a pandas `iter_index` Series and a Python datetime lookup dict, and + multi-minute `get_bars()` resampling slices the cached OHLCV/corporate-action + frame instead of rebuilding a full dataline DataFrame with unused quote + columns. +- **Order reconciliation now reduces avoidable duplicate work for all brokers.** + Strategy order sync indexes LumiBot orders by broker identifier once per + reconciliation pass instead of scanning the full order list for each broker + order. + ### Fixed - **Tradier live polling now throttles repeated broker reads and backs off after transient provider 5xx failures.** Account balances, positions, and orders @@ -14,6 +27,9 @@ position, and order behavior.** The focused tests cover cache reuse after retry-exhausted 5xx responses and ensure OAuth/external-mode polling regressions still pass. +- **Performance regressions now cover large `Data` repair memory, order + identifier indexing, and `get_bars()` resampling with non-OHLCV quote + columns.** ## 4.5.65 - Unreleased diff --git a/lumibot/entities/data.py b/lumibot/entities/data.py index a658c23e9..82fbcb489 100644 --- a/lumibot/entities/data.py +++ b/lumibot/entities/data.py @@ -45,6 +45,7 @@ # PERF: module-level sentinel used to avoid eager-evaluating fallbacks in `getattr()` hot paths. _MISSING = object() +_ITER_INDEX_DICT_MAX_ROWS = 50_000 # Set the option to raise an error if downcasting is not possible (if available in this pandas version) try: @@ -151,6 +152,14 @@ class Data: {"timestep": "minute", "representations": ["1M", "minute"]}, ] + @property + def iter_index(self): + # Backwards-compatible view for callers that inspect the old public attribute. + # Do not retain it on the instance; large backtests already keep `_index_values_ns` + # and, for small frames, `iter_index_dict`. + iter_index = pd.Series(self.df.index) + return pd.Series(iter_index.index, index=iter_index) + def __init__( self, asset, @@ -544,16 +553,6 @@ def repair_times_and_fill(self, idx): except Exception: self._data_len = None - # Set up iter_index and iter_index_dict for later use. - iter_index = pd.Series(df.index) - self.iter_index = pd.Series(iter_index.index, index=iter_index) - # PERF: `to_dict()` produces keys as `pd.Timestamp`, which do not hash-equal to - # `datetime.datetime` objects. Many hot paths pass python datetimes, causing dictionary - # misses and forcing an expensive `Series.asof()` fallback. - # - # Store a second mapping keyed by python datetimes so `dt in iter_index_dict` is fast. - self.iter_index_dict = {ts.to_pydatetime(): int(pos) for ts, pos in self.iter_index.items()} - # PERF: Precompute an integer nanoseconds view of the datetime index so `get_iter_count()` # can use NumPy search/forward cursors without triggering pandas datetime scalar validation. # @@ -585,6 +584,23 @@ def repair_times_and_fill(self, idx): except Exception: self._index_values_ns = None + # Keep exact timestamp dict only for small frames or when nanosecond search fallback is + # unavailable. On large minute datasets this dict can retain tens/hundreds of MB of Python + # datetime objects while `_index_values_ns` already provides equivalent lookup semantics. + try: + if self._data_len is None: + self._data_len = int(len(self.df.index)) + build_iter_dict = self._index_values_ns is None or int(self._data_len) <= _ITER_INDEX_DICT_MAX_ROWS + if build_iter_dict: + self.iter_index_dict = { + (ts.to_pydatetime() if hasattr(ts, "to_pydatetime") else ts): int(pos) + for pos, ts in enumerate(self.df.index) + } + else: + self.iter_index_dict = {} + except Exception: + self.iter_index_dict = {} + # Reset the per-series cursor used by `get_iter_count()` (safe; backtests are single-threaded). self._iter_count_cursor_ns = None self._iter_count_cursor_i = 0 @@ -1220,6 +1236,81 @@ def _get_bars_between_dates_dict(self, timestep=None, start_date=None, end_date= return dict + @check_data + def _validate_bars_request(self, dt, length=1, timestep=None, timeshift=0): + """Run the standard data availability checks without materializing bar data.""" + return True + + def _normalize_timeshift_to_rows(self, timeshift): + if timeshift is None: + return 0 + + if isinstance(timeshift, datetime.timedelta): + if self.timestep == "day": + return int(timeshift.total_seconds() / (24 * 3600)) + if self.timestep == "hour": + return int(timeshift.total_seconds() / 3600) + return int(timeshift.total_seconds() / 60) + + return int(timeshift or 0) + + def _get_bars_row_bounds(self, dt, length=1, timeshift=0): + """Return integer row bounds matching `_get_bars_dict()` slice semantics.""" + timeshift = self._normalize_timeshift_to_rows(timeshift) + + iter_count = self.get_iter_count(dt) + try: + if pd.isna(iter_count): + iter_count = 0 + except Exception: + pass + + # `_get_bars_dict()` slices with `end_row` as an exclusive bound. Daily bars are already + # complete for intraday requests, so include the as-of daily row. + if self.timestep == "day": + end_row = int(iter_count) + 1 - timeshift + else: + end_row = int(iter_count) - timeshift + + data_len = getattr(self, "_data_len", None) + if data_len is None: + data_len = len(next(iter(self.datalines.values())).dataline) if self.datalines else len(self.df.index) + self._data_len = int(data_len) + + end_row = max(0, min(end_row, int(data_len))) + start_row = max(0, end_row - int(length)) + if start_row > end_row: + start_row = end_row + if start_row == end_row and end_row > 0: + start_row = max(0, end_row - 1) + + return int(start_row), int(end_row), int(timeshift) + + def _get_bars_source_frame(self): + """Return the cached OHLCV/corporate-action frame used by `get_bars()`.""" + df_source = getattr(self, "_bars_df", None) + if df_source is not None: + return df_source + + try: + bars_cols = getattr(self, "_bars_cols", None) + df_source = self.df[bars_cols].copy(deep=False) if bars_cols else self.df + if bars_cols: + self._bars_df = df_source + except Exception: + df_source = self.df + + return df_source + + def _get_bars_frame_window(self, dt, length=1, timeshift=0): + """Slice the cached bars frame without building an intermediate dataline DataFrame.""" + start_row, end_row, normalized_timeshift = self._get_bars_row_bounds(dt, length=length, timeshift=timeshift) + df_source = self._get_bars_source_frame() + df = df_source._slice(slice(start_row, end_row)) + if df is None or df.shape[0] == 0: + return None, start_row, end_row, normalized_timeshift + return df, start_row, end_row, normalized_timeshift + def get_bars(self, dt, length=1, timestep=MIN_TIMESTEP, timeshift=0): """Returns a dataframe of the data. @@ -1272,37 +1363,13 @@ def get_bars(self, dt, length=1, timestep=MIN_TIMESTEP, timeshift=0): and int(native_qty) == int(quantity) and native_unit == "minute" ): - try: - iter_count = self.get_iter_count(dt) - if pd.isna(iter_count): - iter_count = 0 - except Exception: - iter_count = self.get_iter_count(dt) - - df_source = getattr(self, "_bars_df", None) - if df_source is None: - try: - bars_cols = getattr(self, "_bars_cols", None) - df_source = self.df[bars_cols].copy(deep=False) if bars_cols else self.df - if bars_cols: - self._bars_df = df_source - except Exception: - df_source = self.df - - if isinstance(timeshift, datetime.timedelta): - timeshift = int(timeshift.total_seconds() / 60) - - end_row = int(iter_count) - int(timeshift or 0) - data_len = getattr(self, "_data_len", None) - if data_len is None: - data_len = int(len(df_source.index)) - self._data_len = data_len - end_row = max(0, min(end_row, data_len)) - start_row = max(0, end_row - int(num_periods)) - if start_row > end_row: - start_row = end_row - if start_row == end_row and end_row > 0: - start_row = max(0, end_row - 1) + df, start_row, end_row, timeshift = self._get_bars_frame_window( + dt, + length=num_periods, + timeshift=timeshift, + ) + if df is None: + return None # PERF: Many strategies request multi-minute history every minute (e.g., 15m SMA while # running on a 1m cadence). When the "current" native bar has not advanced, the @@ -1322,13 +1389,6 @@ def get_bars(self, dt, length=1, timestep=MIN_TIMESTEP, timeshift=0): if cached_df is not None and cached_df.shape[0] != 0: return cached_df - # PERF: `.iloc[start:end]` goes through the indexer stack (`_iLocIndexer`) which - # performs validation on every call. In backtesting we already operate on integer - # row bounds; `_slice()` is the internal fast-path that avoids the indexer overhead. - df = df_source._slice(slice(start_row, end_row)) - if df is None or df.shape[0] == 0: - return None - # PERF: avoid `col in df.columns` membership checks (`Index.__contains__`) on every call. has_volume = getattr(self, "_bars_has_volume", _MISSING) if has_volume is _MISSING: @@ -1396,46 +1456,9 @@ def get_bars(self, dt, length=1, timestep=MIN_TIMESTEP, timeshift=0): # PERF: avoid reconstructing a DataFrame from datalines on every call. # The underlying `self.df` is already indexed by datetime, so we can slice by # row bounds in O(1) and return a stable OHLCV schema. - try: - iter_count = self.get_iter_count(dt) - if pd.isna(iter_count): - iter_count = 0 - except Exception: - iter_count = self.get_iter_count(dt) - - df_source = getattr(self, "_bars_df", None) - if df_source is None: - try: - bars_cols = getattr(self, "_bars_cols", None) - df_source = self.df[bars_cols].copy(deep=False) if bars_cols else self.df - if bars_cols: - self._bars_df = df_source - except Exception: - df_source = self.df - - if isinstance(timeshift, datetime.timedelta): - if self.timestep == "day": - timeshift = int(timeshift.total_seconds() / (24 * 3600)) - elif self.timestep == "hour": - timeshift = int(timeshift.total_seconds() / 3600) - else: - timeshift = int(timeshift.total_seconds() / 60) - - if self.timestep == "day": - end_row = int(iter_count) + 1 - int(timeshift or 0) - else: - end_row = int(iter_count) - int(timeshift or 0) - - data_len = getattr(self, "_data_len", None) - if data_len is None: - data_len = int(len(df_source.index)) - self._data_len = data_len - end_row = max(0, min(end_row, data_len)) - start_row = max(0, end_row - int(length)) - if start_row > end_row: - start_row = end_row - if start_row == end_row and end_row > 0: - start_row = max(0, end_row - 1) + df, start_row, end_row, timeshift = self._get_bars_frame_window(dt, length=length, timeshift=timeshift) + if df is None: + return None # PERF: Cache the last native slice. This is particularly effective for `timestep="day"` # requests when strategies run on an intraday cadence: the daily window only changes @@ -1454,13 +1477,6 @@ def get_bars(self, dt, length=1, timestep=MIN_TIMESTEP, timeshift=0): if cached_df is not None and cached_df.shape[0] != 0: return cached_df - # PERF: `.iloc[start:end]` goes through the indexer stack (`_iLocIndexer`) which - # performs validation on every call. In backtesting we already operate on integer - # row bounds; `_slice()` is the internal fast-path that avoids the indexer overhead. - df = df_source._slice(slice(start_row, end_row)) - if df is None or df.shape[0] == 0: - return None - # PERF: avoid `col in df.columns` membership checks (`Index.__contains__`) on every call. has_volume = getattr(self, "_bars_has_volume", _MISSING) if has_volume is _MISSING: @@ -1507,56 +1523,45 @@ def get_bars(self, dt, length=1, timestep=MIN_TIMESTEP, timeshift=0): # If the data is minute data and we are requesting daily data then multiply the length by 1440 length = length * 1440 unit = "D" - data = self._get_bars_dict(dt, length=length, timestep="minute", timeshift=timeshift) + strict_request_timestep = None elif timestep == "day" and self.timestep == "hour": # If the data is hourly data and we are requesting daily data then multiply the length by 24 length = length * 24 unit = "D" - data = self._get_bars_dict(dt, length=length, timestep="hour", timeshift=timeshift) + strict_request_timestep = None elif timestep == 'day' and self.timestep == 'day': unit = "D" - data = self._get_bars_dict(dt, length=length, timestep=timestep, timeshift=timeshift) + strict_request_timestep = None elif timestep == "hour" and self.timestep == "minute": # Convert requested hours to minutes to pull enough base data for resample. length = length * 60 * quantity unit = "h" - data = self._get_bars_dict( - dt, - length=length, - timestep="minute", - timeshift=timeshift, - _strict_request_timestep=f"{int(quantity)}{timestep}", - ) + strict_request_timestep = f"{int(quantity)}{timestep}" elif timestep == "hour" and self.timestep == "hour": unit = "h" length = length * quantity - data = self._get_bars_dict( - dt, - length=length, - timestep="hour", - timeshift=timeshift, - _strict_request_timestep=f"{int(quantity)}{timestep}" if int(quantity) > 1 else None, - ) + strict_request_timestep = f"{int(quantity)}{timestep}" if int(quantity) > 1 else None else: unit = "min" # Guaranteed to be minute timestep at this point length = length * quantity - data = self._get_bars_dict( - dt, - length=length, - timestep=timestep, - timeshift=timeshift, - _strict_request_timestep=f"{int(quantity)}{timestep}" if int(quantity) > 1 else None, - ) - - if data is None: + strict_request_timestep = f"{int(quantity)}{timestep}" if int(quantity) > 1 else None + + self._validate_bars_request( + dt, + length=length, + timestep=timestep, + timeshift=timeshift, + _strict_request_timestep=strict_request_timestep, + ) + df, _, _, _ = self._get_bars_frame_window(dt, length=length, timeshift=timeshift) + if df is None: return None - df = pd.DataFrame(data).assign(datetime=lambda df: pd.to_datetime(df['datetime'])).set_index('datetime') if "dividend" in df.columns: agg_column_map["dividend"] = "sum" if "stock_splits" in df.columns: diff --git a/lumibot/strategies/strategy_executor.py b/lumibot/strategies/strategy_executor.py index fa1e065f9..bfba1a39f 100644 --- a/lumibot/strategies/strategy_executor.py +++ b/lumibot/strategies/strategy_executor.py @@ -19,7 +19,6 @@ from apscheduler.triggers.cron import CronTrigger from lumibot.constants import LUMIBOT_DEFAULT_PYTZ from lumibot.entities import Asset, Order -from lumibot.entities import Asset from lumibot.strategies.scheduled_timing import ScheduledRunTiming from lumibot.tools import append_locals, get_trading_days, staticdecorator from lumibot.tools.smart_limit_utils import ( @@ -409,12 +408,13 @@ def sync_broker(self): orders_broker = [order for order in orders_broker if order is not None] if len(orders_broker) > 0 or self.broker.get_all_orders(): orders_lumi = self.broker.get_all_orders() + orders_lumi_by_identifier = self._index_orders_by_identifier(orders_lumi) # Check orders at the broker against those in lumibot. for order in orders_broker: # Check against existing orders. - order_lumi = [ord_lumi for ord_lumi in orders_lumi if ord_lumi.identifier == order.identifier] - if len(order_lumi) > 1: + order_lumi = orders_lumi_by_identifier.get(order.identifier) + if isinstance(order_lumi, list): self.strategy.logger.warning( f"Multiple orders found in lumibot with the same identifier {order.identifier}. " f"This should not happen and indicates a bug in the order tracking. This is manifesting as " @@ -423,8 +423,6 @@ def sync_broker(self): f"Orders: {order_lumi}" ) order_lumi = self.broker._clean_order_trackers(order) - else: - order_lumi = order_lumi[0] if len(order_lumi) > 0 else None if order_lumi: # Compare the orders. @@ -563,6 +561,20 @@ def sync_broker(self): self.broker._hold_trade_events = False self.broker.process_held_trades() + @staticmethod + def _index_orders_by_identifier(orders: list[Order]) -> dict: + orders_by_identifier = {} + for order in orders: + identifier = order.identifier + existing = orders_by_identifier.get(identifier) + if existing is None: + orders_by_identifier[identifier] = order + elif isinstance(existing, list): + existing.append(order) + else: + orders_by_identifier[identifier] = [existing, order] + return orders_by_identifier + @staticmethod def _get_all_order_identifiers(orders_broker: list[Order]) -> set: """ diff --git a/tests/test_data_entity.py b/tests/test_data_entity.py index efb1a6c01..af45ebdbe 100644 --- a/tests/test_data_entity.py +++ b/tests/test_data_entity.py @@ -326,6 +326,38 @@ def test_strict_intraday_rejects_multi_minute_history_past_bucket_tolerance(self with pytest.raises(ValueError, match="after the available data's end"): data.get_bars(base_dt + timedelta(minutes=35), length=3, timestep="5m") + def test_get_bars_resample_ignores_non_ohlcv_columns(self): + asset = Asset("SPY") + tz = pytz.timezone("America/New_York") + base_dt = tz.localize(datetime(2026, 6, 23, 9, 30)) + dates = [base_dt + timedelta(minutes=i) for i in range(12)] + df = ( + pd.DataFrame( + { + "datetime": dates, + "open": [100.0 + i for i in range(12)], + "high": [101.0 + i for i in range(12)], + "low": [99.0 + i for i in range(12)], + "close": [100.5 + i for i in range(12)], + "volume": [10.0] * 12, + "bid": [100.25 + i for i in range(12)], + "ask": [100.75 + i for i in range(12)], + "last_trade_time": dates, + } + ) + .set_index("datetime") + ) + data = Data(asset, df, timestep="minute") + expected_data = Data(asset, df[["open", "high", "low", "close", "volume"]].copy(), timestep="minute") + + bars = data.get_bars(base_dt + timedelta(minutes=11), length=2, timestep="5m") + expected = expected_data.get_bars(base_dt + timedelta(minutes=11), length=2, timestep="5m") + + assert bars is not None + assert expected is not None + assert list(bars.columns) == ["open", "high", "low", "close", "volume"] + pd.testing.assert_frame_equal(bars, expected) + def test_get_quote_includes_source_bar_provenance(self): asset = Asset("BTC", asset_type=Asset.AssetType.CRYPTO) tz = pytz.timezone("America/New_York") diff --git a/tests/test_memory_efficiency_entities_backtesting.py b/tests/test_memory_efficiency_entities_backtesting.py new file mode 100644 index 000000000..04ad542a7 --- /dev/null +++ b/tests/test_memory_efficiency_entities_backtesting.py @@ -0,0 +1,34 @@ +import datetime as dt + +import pandas as pd + +from lumibot.entities import Asset +from lumibot.entities.data import Data + + +def _ohlcv_frame(index: pd.DatetimeIndex) -> pd.DataFrame: + row_count = len(index) + return pd.DataFrame( + { + "open": range(row_count), + "high": range(row_count), + "low": range(row_count), + "close": range(row_count), + "volume": 1, + }, + index=index, + ) + + +def test_large_data_repair_avoids_retained_iter_index_dict_and_preserves_lookup(): + index = pd.date_range("2024-01-01", periods=50_001, freq="min", tz="America/New_York") + data = Data(Asset("MEM"), _ohlcv_frame(index), timestep="minute") + + data.repair_times_and_fill(index) + + assert data.iter_index_dict == {} + assert data.get_iter_count(index[-1].to_pydatetime()) == len(index) - 1 + assert data.get_iter_count(index[123].to_pydatetime() + dt.timedelta(seconds=30)) == 123 + + lazy_iter_index = data.iter_index + assert lazy_iter_index.loc[index[123]] == 123 diff --git a/tests/test_strategy_executor_order_index.py b/tests/test_strategy_executor_order_index.py new file mode 100644 index 000000000..7fd3ba3da --- /dev/null +++ b/tests/test_strategy_executor_order_index.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from lumibot.strategies.strategy_executor import StrategyExecutor + + +def test_index_orders_by_identifier_returns_direct_order_for_unique_ids(): + first = SimpleNamespace(identifier="1") + second = SimpleNamespace(identifier="2") + + indexed = StrategyExecutor._index_orders_by_identifier([first, second]) + + assert indexed == {"1": first, "2": second} + + +def test_index_orders_by_identifier_preserves_duplicates_for_cleanup_path(): + first = SimpleNamespace(identifier="1") + duplicate = SimpleNamespace(identifier="1") + other = SimpleNamespace(identifier="2") + + indexed = StrategyExecutor._index_orders_by_identifier([first, duplicate, other]) + + assert indexed["1"] == [first, duplicate] + assert indexed["2"] is other From ec56ea662493d1e5c548dfab69b3fe19eeabf03f Mon Sep 17 00:00:00 2001 From: Al4ise Date: Sat, 4 Jul 2026 19:58:15 +0300 Subject: [PATCH 2/3] Improve backtesting data memory reuse --- CHANGELOG.md | 4 +++- lumibot/data_sources/pandas_data.py | 2 +- lumibot/entities/data.py | 17 ++++++++++++++--- ...st_memory_efficiency_entities_backtesting.py | 12 ++++++++++++ ...est_pandas_data_find_asset_timestep_match.py | 13 ++++++++++++- 5 files changed, 42 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 771ae668f..174cbdac4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,9 @@ both a pandas `iter_index` Series and a Python datetime lookup dict, and multi-minute `get_bars()` resampling slices the cached OHLCV/corporate-action frame instead of rebuilding a full dataline DataFrame with unused quote - columns. + columns. Datetime datalines also reuse the existing `DatetimeIndex` backing + array instead of allocating a duplicate object array, and PandasData reuses + the shared default USD quote asset when normalizing data-store keys. - **Order reconciliation now reduces avoidable duplicate work for all brokers.** Strategy order sync indexes LumiBot orders by broker identifier once per reconciliation pass instead of scanning the full order list for each broker diff --git a/lumibot/data_sources/pandas_data.py b/lumibot/data_sources/pandas_data.py index 1f5809e77..eb7e6904c 100644 --- a/lumibot/data_sources/pandas_data.py +++ b/lumibot/data_sources/pandas_data.py @@ -69,7 +69,7 @@ def _get_new_pandas_data_key(data): if data.quote is None: # Warn that USD is being used as the quote logger.warning(f"No quote specified for {data.asset}. Using USD as the quote.") - return data.asset, Asset(symbol="USD", asset_type="forex") + return data.asset, _USD_FOREX return data.asset, data.quote else: raise ValueError("Asset must be an Asset or a tuple of Asset and quote") diff --git a/lumibot/entities/data.py b/lumibot/entities/data.py index 82fbcb489..65591afd5 100644 --- a/lumibot/entities/data.py +++ b/lumibot/entities/data.py @@ -637,17 +637,18 @@ def repair_times_and_fill(self, idx): self._bars_df = None def to_datalines(self): + datetime_values = self.df.index.array self.datalines.update( { "datetime": Dataline( self.asset, "datetime", - self.df.index.to_numpy(), + datetime_values, self.df.index.dtype, ) } ) - self.datetime = self.datalines["datetime"].dataline + self.datetime = datetime_values for column in self.df.columns: self.datalines.update( @@ -1012,7 +1013,17 @@ def get_last_price(self, dt, length=1, timeshift=0) -> Union[float, Decimal, Non if self.timestep == "day": price = close_price else: - price = close_price if dt > self.datalines["datetime"].dataline[iter_count] else open_price + index_ns = getattr(self, "_index_values_ns", None) + dt_ns = getattr(self, "_iter_count_cursor_ns", None) + if index_ns is not None and dt_ns is not None: + try: + # `get_iter_count()` already normalized both values to ns; avoid Timestamp scalar allocation here. + use_close = int(dt_ns) > int(index_ns[int(iter_count)]) + except Exception: + use_close = dt > self.datalines["datetime"].dataline[iter_count] + else: + use_close = dt > self.datalines["datetime"].dataline[iter_count] + price = close_price if use_close else open_price if price is None: return None diff --git a/tests/test_memory_efficiency_entities_backtesting.py b/tests/test_memory_efficiency_entities_backtesting.py index 04ad542a7..b50cabe6c 100644 --- a/tests/test_memory_efficiency_entities_backtesting.py +++ b/tests/test_memory_efficiency_entities_backtesting.py @@ -32,3 +32,15 @@ def test_large_data_repair_avoids_retained_iter_index_dict_and_preserves_lookup( lazy_iter_index = data.iter_index assert lazy_iter_index.loc[index[123]] == 123 + + +def test_data_datetime_dataline_reuses_index_array(): + index = pd.date_range("2024-01-01", periods=10, freq="min", tz="America/New_York") + data = Data(Asset("MEM"), _ohlcv_frame(index), timestep="minute") + data.repair_times_and_fill(index) + + datetime_line = data.datalines["datetime"].dataline + + assert datetime_line is data.df.index.array + assert data.datetime is datetime_line + assert datetime_line[3] == data.df.index[3] diff --git a/tests/test_pandas_data_find_asset_timestep_match.py b/tests/test_pandas_data_find_asset_timestep_match.py index e3aa95a69..7586ca062 100644 --- a/tests/test_pandas_data_find_asset_timestep_match.py +++ b/tests/test_pandas_data_find_asset_timestep_match.py @@ -2,7 +2,7 @@ import pandas as pd -from lumibot.data_sources.pandas_data import PandasData +from lumibot.data_sources.pandas_data import _USD_FOREX, PandasData from lumibot.entities import Asset, Data @@ -16,6 +16,17 @@ def _day_df(tz: str = "America/New_York") -> pd.DataFrame: return pd.DataFrame({"open": [1, 2, 3], "high": [1, 2, 3], "low": [1, 2, 3], "close": [1, 2, 3], "volume": [0, 0, 0]}, index=idx) +def test_set_pandas_data_keys_reuses_default_usd_quote_singleton(): + base = Asset("AAPL", asset_type=Asset.AssetType.STOCK) + data = Data(base, _day_df(), timestep="day") + + keyed = PandasData._set_pandas_data_keys([data]) + key = next(iter(keyed)) + + assert key == (base, Asset("USD", asset_type=Asset.AssetType.FOREX)) + assert key[1] is _USD_FOREX + + def test_find_asset_in_data_store_does_not_return_daily_for_minute_requests(): base = Asset("BTC", asset_type=Asset.AssetType.CRYPTO) quote = Asset("USD", asset_type=Asset.AssetType.FOREX) From 2f977d00e5ccce76972654e401668c4cc5bed0ee Mon Sep 17 00:00:00 2001 From: Al4ise Date: Sat, 4 Jul 2026 21:00:43 +0300 Subject: [PATCH 3/3] Address backtesting data review feedback --- lumibot/entities/data.py | 2 +- lumibot/strategies/_strategy.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lumibot/entities/data.py b/lumibot/entities/data.py index 65591afd5..75e789b78 100644 --- a/lumibot/entities/data.py +++ b/lumibot/entities/data.py @@ -1317,7 +1317,7 @@ def _get_bars_frame_window(self, dt, length=1, timeshift=0): """Slice the cached bars frame without building an intermediate dataline DataFrame.""" start_row, end_row, normalized_timeshift = self._get_bars_row_bounds(dt, length=length, timeshift=timeshift) df_source = self._get_bars_source_frame() - df = df_source._slice(slice(start_row, end_row)) + df = df_source.iloc[start_row:end_row] if df is None or df.shape[0] == 0: return None, start_row, end_row, normalized_timeshift return df, start_row, end_row, normalized_timeshift diff --git a/lumibot/strategies/_strategy.py b/lumibot/strategies/_strategy.py index 6287f71b9..443bf3cdc 100644 --- a/lumibot/strategies/_strategy.py +++ b/lumibot/strategies/_strategy.py @@ -36,8 +36,8 @@ CcxtBacktesting, DataBentoDataBacktesting, InteractiveBrokersRESTBacktesting, - PolymarketBacktesting, PolygonDataBacktesting, + PolymarketBacktesting, RoutedBacktestingPandas, ThetaDataBacktesting, ThetaDataBacktestingPandas,