Document build_full_result contract and log dropped members

botocore/paginate.py

@@ -483,7 +483,16 @@ def result_key_iters(self):
         ]
 
     def build_full_result(self):
+        """Aggregate all pages into a single combined result.
+
+        ``result_key`` values are concatenated/summed across pages and
+        ``non_aggregate_keys`` are taken from the first page (assumed static).
+        Other top-level members are not guaranteed and are generally dropped;
+        iterate the pages directly if you need them.
+        """
         complete_result = {}
+        # Last page's top-level members, for the dropped-member debug log.
+        last_page_keys = set()
         for response in self:
             page = response
             # We want to try to catch operation object pagination
@@ -493,6 +502,8 @@ def build_full_result(self):
             # uses. We can remove it though once operation objects are removed.
             if isinstance(response, tuple) and len(response) == 2:
                 page = response[1]
+            if isinstance(page, dict):
+                last_page_keys = set(page)
             # We're incrementally building the full response page
             # by page.  For each page in the response we need to
             # inject the necessary components from the page
@@ -529,6 +540,19 @@ def build_full_result(self):
         merge_dicts(complete_result, self.non_aggregate_part)
         if self.resume_token is not None:
             complete_result['NextToken'] = self.resume_token
+        # Log top-level members that weren't aggregated (not a result_key or
+        # non_aggregate_key) and so were dropped from the combined result.
+        if log.isEnabledFor(logging.DEBUG):
+            dropped = last_page_keys - set(complete_result)
+            dropped.discard('ResponseMetadata')  # never part of paginated data
+            if dropped:
+                log.debug(
+                    "The following top-level output members are not "
+                    "accounted for in the pagination config and were dropped "
+                    "from build_full_result: %s. Iterate the pages directly "
+                    "if you need these values.",
+                    ', '.join(sorted(dropped)),
+                )
         return complete_result
 
     def _parse_starting_token(self):

tests/unit/test_paginate.py

@@ -332,6 +332,49 @@ def test_build_full_result_with_single_key(self):
         complete = pages.build_full_result()
         self.assertEqual(complete, {'Users': ['User1', 'User2', 'User3']})
 
+    def test_build_full_result_drops_unaccounted_members(self):
+        # Members that are neither a result_key nor a non_aggregate_key are
+        # not carried into the combined result.
+        self.paginate_config = {
+            "output_token": "Marker",
+            "input_token": "Marker",
+            "result_key": "Users",
+        }
+        self.paginator = Paginator(
+            self.method, self.paginate_config, self.model
+        )
+        responses = [
+            {"Users": ["User1"], "Marker": "m1", "Status": "OK"},
+            {"Users": ["User2"], "Status": "OK"},
+        ]
+        self.method.side_effect = responses
+        pages = self.paginator.paginate()
+        complete = pages.build_full_result()
+        self.assertEqual(complete, {'Users': ['User1', 'User2']})
+        self.assertNotIn('Status', complete)
+
+    def test_build_full_result_logs_dropped_members(self):
+        self.paginate_config = {
+            "output_token": "Marker",
+            "input_token": "Marker",
+            "result_key": "Users",
+        }
+        self.paginator = Paginator(
+            self.method, self.paginate_config, self.model
+        )
+        responses = [
+            {"Users": ["User1"], "Status": "OK", "ResponseMetadata": {}},
+        ]
+        self.method.side_effect = responses
+        pages = self.paginator.paginate()
+        with self.assertLogs('botocore.paginate', level='DEBUG') as log_cm:
+            pages.build_full_result()
+        messages = '\n'.join(log_cm.output)
+        # The unaccounted member is reported...
+        self.assertIn('Status', messages)
+        # ...but ResponseMetadata is not flagged as a dropped member.
+        self.assertNotIn('ResponseMetadata', messages)
+
     def test_build_multiple_results(self):
         self.paginate_config = {
             "output_token": "Marker",