diff --git a/botocore/paginate.py b/botocore/paginate.py index 63b4a29f80..342577f0fe 100644 --- a/botocore/paginate.py +++ b/botocore/paginate.py @@ -483,7 +483,16 @@ def result_key_iters(self): ] def build_full_result(self): + """Aggregate all pages into a single combined result. + + ``result_key`` values are concatenated/summed across pages and + ``non_aggregate_keys`` are taken from the first page (assumed static). + Other top-level members are not guaranteed and are generally dropped; + iterate the pages directly if you need them. + """ complete_result = {} + # Last page's top-level members, for the dropped-member debug log. + last_page_keys = set() for response in self: page = response # We want to try to catch operation object pagination @@ -493,6 +502,8 @@ def build_full_result(self): # uses. We can remove it though once operation objects are removed. if isinstance(response, tuple) and len(response) == 2: page = response[1] + if isinstance(page, dict): + last_page_keys = set(page) # We're incrementally building the full response page # by page. For each page in the response we need to # inject the necessary components from the page @@ -529,6 +540,19 @@ def build_full_result(self): merge_dicts(complete_result, self.non_aggregate_part) if self.resume_token is not None: complete_result['NextToken'] = self.resume_token + # Log top-level members that weren't aggregated (not a result_key or + # non_aggregate_key) and so were dropped from the combined result. + if log.isEnabledFor(logging.DEBUG): + dropped = last_page_keys - set(complete_result) + dropped.discard('ResponseMetadata') # never part of paginated data + if dropped: + log.debug( + "The following top-level output members are not " + "accounted for in the pagination config and were dropped " + "from build_full_result: %s. Iterate the pages directly " + "if you need these values.", + ', '.join(sorted(dropped)), + ) return complete_result def _parse_starting_token(self): diff --git a/tests/unit/test_paginate.py b/tests/unit/test_paginate.py index 470cd1eecf..b18df3ff14 100644 --- a/tests/unit/test_paginate.py +++ b/tests/unit/test_paginate.py @@ -332,6 +332,49 @@ def test_build_full_result_with_single_key(self): complete = pages.build_full_result() self.assertEqual(complete, {'Users': ['User1', 'User2', 'User3']}) + def test_build_full_result_drops_unaccounted_members(self): + # Members that are neither a result_key nor a non_aggregate_key are + # not carried into the combined result. + self.paginate_config = { + "output_token": "Marker", + "input_token": "Marker", + "result_key": "Users", + } + self.paginator = Paginator( + self.method, self.paginate_config, self.model + ) + responses = [ + {"Users": ["User1"], "Marker": "m1", "Status": "OK"}, + {"Users": ["User2"], "Status": "OK"}, + ] + self.method.side_effect = responses + pages = self.paginator.paginate() + complete = pages.build_full_result() + self.assertEqual(complete, {'Users': ['User1', 'User2']}) + self.assertNotIn('Status', complete) + + def test_build_full_result_logs_dropped_members(self): + self.paginate_config = { + "output_token": "Marker", + "input_token": "Marker", + "result_key": "Users", + } + self.paginator = Paginator( + self.method, self.paginate_config, self.model + ) + responses = [ + {"Users": ["User1"], "Status": "OK", "ResponseMetadata": {}}, + ] + self.method.side_effect = responses + pages = self.paginator.paginate() + with self.assertLogs('botocore.paginate', level='DEBUG') as log_cm: + pages.build_full_result() + messages = '\n'.join(log_cm.output) + # The unaccounted member is reported... + self.assertIn('Status', messages) + # ...but ResponseMetadata is not flagged as a dropped member. + self.assertNotIn('ResponseMetadata', messages) + def test_build_multiple_results(self): self.paginate_config = { "output_token": "Marker",