Skip to content

Add relative percentile degradation and error count ceiling criteria to AutoStop listener #883

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
undera merged 4 commits into from
Sep 2, 2026
Merged

Add relative percentile degradation and error count ceiling criteria to AutoStop listener #883

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
198f505
Add relative window-to-window percentile and error count checks in Au…
hilalsidhic Sep 2, 2026
e393e69
Add JMeter test plans for AutoStop error count and percentile checks
hilalsidhic Sep 2, 2026
ab2a012
Enhance AutoStop functionality with relative window percentile degrad…
hilalsidhic Sep 2, 2026
802c77a
Merge branch 'master' into mhilal/auto-stop-relative-window
undera Sep 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 145 additions & 0 deletions examples/AutoStopExample_ErrCount.jmx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
<?xml version="1.0" encoding="UTF-8"?>
<jmeterTestPlan version="1.2" properties="5.0" jmeter="5.6.3">
<hashTree>
<TestPlan guiclass="TestPlanGui" testclass="TestPlan" testname="Test Plan" enabled="true">
<stringProp name="TestPlan.comments">This Test Plan demonstrates how AutoStop is used to terminate a test
when the absolute error count in a window exceeds a threshold.</stringProp>
<boolProp name="TestPlan.functional_mode">false</boolProp>
<boolProp name="TestPlan.serialize_threadgroups">false</boolProp>
<elementProp name="TestPlan.user_defined_variables" elementType="Arguments" guiclass="ArgumentsPanel" testclass="Arguments" testname="User Defined Variables" enabled="true">
<collectionProp name="Arguments.arguments"/>
</elementProp>
<stringProp name="TestPlan.user_define_classpath"></stringProp>
</TestPlan>
<hashTree>
<kg.apc.jmeter.reporters.AutoStop guiclass="kg.apc.jmeter.reporters.AutoStopGui" testclass="kg.apc.jmeter.reporters.AutoStop" testname="jp@gc - AutoStop Listener">
<stringProp name="avg_response_time"></stringProp>
<stringProp name="avg_response_time_length"></stringProp>
<stringProp name="error_rate"></stringProp>
<stringProp name="error_rate_length"></stringProp>
<stringProp name="avg_response_latency"></stringProp>
<stringProp name="avg_response_latency_length"></stringProp>
<stringProp name="percentile_response_time"></stringProp>
<stringProp name="percentile_response_time_secs"></stringProp>
<stringProp name="percentile_value"></stringProp>
<stringProp name="rel_percentile_value"></stringProp>
<stringProp name="rel_percentile_window_secs"></stringProp>
<stringProp name="rel_percentile_threshold_pct"></stringProp>
<stringProp name="error_count">15</stringProp>
<stringProp name="error_count_length">5</stringProp>
<stringProp name="TestPlan.comments">Stops the test if more than 15 errors occur within a 5-second window.</stringProp>
<stringProp name="custom_validation_duration"></stringProp>
</kg.apc.jmeter.reporters.AutoStop>
<hashTree/>
<kg.apc.jmeter.vizualizers.CorrectedResultCollector guiclass="kg.apc.jmeter.vizualizers.TransactionsPerSecondGui" testclass="kg.apc.jmeter.vizualizers.CorrectedResultCollector" testname="jp@gc - Transactions per Second">
<boolProp name="ResultCollector.error_logging">false</boolProp>
<objProp>
<name>saveConfig</name>
<value class="SampleSaveConfiguration">
<time>true</time>
<latency>true</latency>
<timestamp>true</timestamp>
<success>true</success>
<label>true</label>
<code>true</code>
<message>true</message>
<threadName>true</threadName>
<dataType>true</dataType>
<encoding>false</encoding>
<assertions>true</assertions>
<subresults>true</subresults>
<responseData>false</responseData>
<samplerData>false</samplerData>
<xml>false</xml>
<fieldNames>false</fieldNames>
<responseHeaders>false</responseHeaders>
<requestHeaders>false</requestHeaders>
<responseDataOnError>false</responseDataOnError>
<saveAssertionResultsFailureMessage>false</saveAssertionResultsFailureMessage>
<assertionsResultsToSave>0</assertionsResultsToSave>
<bytes>true</bytes>
<threadCounts>true</threadCounts>
</value>
</objProp>
<stringProp name="filename"></stringProp>
<longProp name="interval_grouping">1000</longProp>
<boolProp name="graph_aggregated">false</boolProp>
<stringProp name="include_sample_labels"></stringProp>
<stringProp name="exclude_sample_labels"></stringProp>
<stringProp name="start_offset"></stringProp>
<stringProp name="end_offset"></stringProp>
<boolProp name="include_checkbox_state">false</boolProp>
<boolProp name="exclude_checkbox_state">false</boolProp>
<stringProp name="TestPlan.comments">After 10 seconds of successful requests, errors start occurring.
When error count exceeds 15 in the 5-second window, AutoStop terminates the test.</stringProp>
</kg.apc.jmeter.vizualizers.CorrectedResultCollector>
<hashTree/>
<kg.apc.jmeter.threads.UltimateThreadGroup guiclass="kg.apc.jmeter.threads.UltimateThreadGroupGui" testclass="kg.apc.jmeter.threads.UltimateThreadGroup" testname="jp@gc - Ultimate Thread Group Success" enabled="true">
<collectionProp name="ultimatethreadgroupdata">
<collectionProp name="415028672">
<stringProp name="53">2</stringProp>
<stringProp name="0">0</stringProp>
<stringProp name="48">0</stringProp>
<stringProp name="1722">60</stringProp>
<stringProp name="48">0</stringProp>
</collectionProp>
</collectionProp>
<elementProp name="ThreadGroup.main_controller" elementType="LoopController" guiclass="LoopControlPanel" testclass="LoopController" testname="Loop Controller">
<intProp name="LoopController.loops">-1</intProp>
<boolProp name="LoopController.continue_forever">false</boolProp>
</elementProp>
<stringProp name="ThreadGroup.on_sample_error">continue</stringProp>
<stringProp name="TestPlan.comments">Successful traffic: 2 threads with 200 OK responses</stringProp>
</kg.apc.jmeter.threads.UltimateThreadGroup>
<hashTree>
<kg.apc.jmeter.samplers.DummySampler guiclass="kg.apc.jmeter.samplers.DummySamplerGui" testclass="kg.apc.jmeter.samplers.DummySampler" testname="jp@gc - Dummy Sampler 200" enabled="true">
<boolProp name="WAITING">true</boolProp>
<boolProp name="SUCCESFULL">true</boolProp>
<stringProp name="RESPONSE_CODE">200</stringProp>
<stringProp name="RESPONSE_MESSAGE">OK</stringProp>
<stringProp name="RESPONSE_DATA">Successful response</stringProp>
<stringProp name="RESPONSE_TIME">100</stringProp>
<stringProp name="LATENCY">20</stringProp>
<stringProp name="CONNECT">5</stringProp>
<stringProp name="URL">http://dummy</stringProp>
<stringProp name="RESULT_CLASS">org.apache.jmeter.samplers.SampleResult</stringProp>
<stringProp name="REQUEST_DATA"></stringProp>
</kg.apc.jmeter.samplers.DummySampler>
<hashTree/>
</hashTree>
<kg.apc.jmeter.threads.UltimateThreadGroup guiclass="kg.apc.jmeter.threads.UltimateThreadGroupGui" testclass="kg.apc.jmeter.threads.UltimateThreadGroup" testname="jp@gc - Ultimate Thread Group Failures" enabled="true">
<collectionProp name="ultimatethreadgroupdata">
<collectionProp name="1054589546">
<stringProp name="53">5</stringProp>
<stringProp name="1567">10</stringProp>
<stringProp name="48">0</stringProp>
<stringProp name="1722">50</stringProp>
<stringProp name="48">0</stringProp>
</collectionProp>
</collectionProp>
<elementProp name="ThreadGroup.main_controller" elementType="LoopController" guiclass="LoopControlPanel" testclass="LoopController" testname="Loop Controller">
<intProp name="LoopController.loops">-1</intProp>
<boolProp name="LoopController.continue_forever">false</boolProp>
</elementProp>
<stringProp name="ThreadGroup.on_sample_error">continue</stringProp>
<stringProp name="TestPlan.comments">Simulates errors after 10s: 5 threads generating 500 errors</stringProp>
</kg.apc.jmeter.threads.UltimateThreadGroup>
<hashTree>
<kg.apc.jmeter.samplers.DummySampler guiclass="kg.apc.jmeter.samplers.DummySamplerGui" testclass="kg.apc.jmeter.samplers.DummySampler" testname="jp@gc - Dummy Sampler 500" enabled="true">
<boolProp name="WAITING">true</boolProp>
<boolProp name="SUCCESFULL">false</boolProp>
<stringProp name="RESPONSE_CODE">500</stringProp>
<stringProp name="RESPONSE_MESSAGE">Internal Server Error</stringProp>
<stringProp name="RESPONSE_DATA">Error response</stringProp>
<stringProp name="RESPONSE_TIME">100</stringProp>
<stringProp name="LATENCY">20</stringProp>
<stringProp name="CONNECT">5</stringProp>
<stringProp name="URL">http://dummy</stringProp>
<stringProp name="RESULT_CLASS">org.apache.jmeter.samplers.SampleResult</stringProp>
<stringProp name="REQUEST_DATA"></stringProp>
</kg.apc.jmeter.samplers.DummySampler>
<hashTree/>
</hashTree>
</hashTree>
</hashTree>
</jmeterTestPlan>
145 changes: 145 additions & 0 deletions examples/AutoStopExample_Percentile.jmx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
<?xml version="1.0" encoding="UTF-8"?>
<jmeterTestPlan version="1.2" properties="5.0" jmeter="5.6.3">
<hashTree>
<TestPlan guiclass="TestPlanGui" testclass="TestPlan" testname="Test Plan" enabled="true">
<stringProp name="TestPlan.comments">This Test Plan demonstrates how AutoStop is used to terminate a test
when response time percentile degrades significantly compared to the previous window.</stringProp>
<boolProp name="TestPlan.functional_mode">false</boolProp>
<boolProp name="TestPlan.serialize_threadgroups">false</boolProp>
<elementProp name="TestPlan.user_defined_variables" elementType="Arguments" guiclass="ArgumentsPanel" testclass="Arguments" testname="User Defined Variables" enabled="true">
<collectionProp name="Arguments.arguments"/>
</elementProp>
<stringProp name="TestPlan.user_define_classpath"></stringProp>
</TestPlan>
<hashTree>
<kg.apc.jmeter.reporters.AutoStop guiclass="kg.apc.jmeter.reporters.AutoStopGui" testclass="kg.apc.jmeter.reporters.AutoStop" testname="jp@gc - AutoStop Listener">
<stringProp name="avg_response_time"></stringProp>
<stringProp name="avg_response_time_length"></stringProp>
<stringProp name="error_rate"></stringProp>
<stringProp name="error_rate_length"></stringProp>
<stringProp name="avg_response_latency"></stringProp>
<stringProp name="avg_response_latency_length"></stringProp>
<stringProp name="percentile_response_time"></stringProp>
<stringProp name="percentile_response_time_secs"></stringProp>
<stringProp name="percentile_value"></stringProp>
<stringProp name="rel_percentile_value">90</stringProp>
<stringProp name="rel_percentile_window_secs">5</stringProp>
<stringProp name="rel_percentile_threshold_pct">50</stringProp>
<stringProp name="error_count"></stringProp>
<stringProp name="error_count_length"></stringProp>
<stringProp name="TestPlan.comments">Stops the test if P90 response time degrades by more than 50%
compared to the previous 5-second window.</stringProp>
<stringProp name="custom_validation_duration"></stringProp>
</kg.apc.jmeter.reporters.AutoStop>
<hashTree/>
<kg.apc.jmeter.vizualizers.CorrectedResultCollector guiclass="kg.apc.jmeter.vizualizers.ResponseTimesOverTimeGui" testclass="kg.apc.jmeter.vizualizers.CorrectedResultCollector" testname="jp@gc - Response Times Over Time">
<boolProp name="ResultCollector.error_logging">false</boolProp>
<objProp>
<name>saveConfig</name>
<value class="SampleSaveConfiguration">
<time>true</time>
<latency>true</latency>
<timestamp>true</timestamp>
<success>true</success>
<label>true</label>
<code>true</code>
<message>true</message>
<threadName>true</threadName>
<dataType>true</dataType>
<encoding>false</encoding>
<assertions>true</assertions>
<subresults>true</subresults>
<responseData>false</responseData>
<samplerData>false</samplerData>
<xml>false</xml>
<fieldNames>false</fieldNames>
<responseHeaders>false</responseHeaders>
<requestHeaders>false</requestHeaders>
<responseDataOnError>false</responseDataOnError>
<saveAssertionResultsFailureMessage>false</saveAssertionResultsFailureMessage>
<assertionsResultsToSave>0</assertionsResultsToSave>
<bytes>true</bytes>
<threadCounts>true</threadCounts>
</value>
</objProp>
<stringProp name="filename"></stringProp>
<longProp name="interval_grouping">500</longProp>
<boolProp name="graph_aggregated">false</boolProp>
<stringProp name="include_sample_labels"></stringProp>
<stringProp name="exclude_sample_labels"></stringProp>
<stringProp name="start_offset"></stringProp>
<stringProp name="end_offset"></stringProp>
<boolProp name="include_checkbox_state">false</boolProp>
<boolProp name="exclude_checkbox_state">false</boolProp>
<stringProp name="TestPlan.comments">Shows response times jumping when high latency threads start at 10s</stringProp>
</kg.apc.jmeter.vizualizers.CorrectedResultCollector>
<hashTree/>
<kg.apc.jmeter.threads.UltimateThreadGroup guiclass="kg.apc.jmeter.threads.UltimateThreadGroupGui" testclass="kg.apc.jmeter.threads.UltimateThreadGroup" testname="jp@gc - Ultimate Thread Group Baseline">
<collectionProp name="ultimatethreadgroupdata">
<collectionProp name="417799235">
<stringProp name="53">5</stringProp>
<stringProp name="0">0</stringProp>
<stringProp name="48">0</stringProp>
<stringProp name="1722">60</stringProp>
<stringProp name="48">0</stringProp>
</collectionProp>
</collectionProp>
<elementProp name="ThreadGroup.main_controller" elementType="LoopController" guiclass="LoopControlPanel" testclass="LoopController" testname="Loop Controller">
<intProp name="LoopController.loops">-1</intProp>
<boolProp name="LoopController.continue_forever">false</boolProp>
</elementProp>
<stringProp name="ThreadGroup.on_sample_error">continue</stringProp>
<stringProp name="TestPlan.comments">Baseline traffic: 5 threads with 50 ms response time</stringProp>
</kg.apc.jmeter.threads.UltimateThreadGroup>
<hashTree>
<kg.apc.jmeter.samplers.DummySampler guiclass="kg.apc.jmeter.samplers.DummySamplerGui" testclass="kg.apc.jmeter.samplers.DummySampler" testname="jp@gc - Dummy Sampler Fast">
<boolProp name="WAITING">true</boolProp>
<boolProp name="SUCCESFULL">true</boolProp>
<stringProp name="RESPONSE_CODE">200</stringProp>
<stringProp name="RESPONSE_MESSAGE">OK</stringProp>
<stringProp name="RESPONSE_DATA">Baseline fast response</stringProp>
<stringProp name="RESPONSE_TIME">50</stringProp>
<stringProp name="LATENCY">20</stringProp>
<stringProp name="CONNECT">5</stringProp>
<stringProp name="URL">http://dummy</stringProp>
<stringProp name="RESULT_CLASS">org.apache.jmeter.samplers.SampleResult</stringProp>
<stringProp name="REQUEST_DATA"></stringProp>
</kg.apc.jmeter.samplers.DummySampler>
<hashTree/>
</hashTree>
<kg.apc.jmeter.threads.UltimateThreadGroup guiclass="kg.apc.jmeter.threads.UltimateThreadGroupGui" testclass="kg.apc.jmeter.threads.UltimateThreadGroup" testname="jp@gc - Ultimate Thread Group Degradation">
<collectionProp name="ultimatethreadgroupdata">
<collectionProp name="-1094036201">
<stringProp name="1567">10</stringProp>
<stringProp name="1567">10</stringProp>
<stringProp name="48">0</stringProp>
<stringProp name="1722">50</stringProp>
<stringProp name="48">0</stringProp>
</collectionProp>
</collectionProp>
<elementProp name="ThreadGroup.main_controller" elementType="LoopController" guiclass="LoopControlPanel" testclass="LoopController" testname="Loop Controller">
<intProp name="LoopController.loops">-1</intProp>
<boolProp name="LoopController.continue_forever">false</boolProp>
</elementProp>
<stringProp name="ThreadGroup.on_sample_error">continue</stringProp>
<stringProp name="TestPlan.comments">Simulates latency degradation after 10s: 10 threads with 500 ms response time</stringProp>
</kg.apc.jmeter.threads.UltimateThreadGroup>
<hashTree>
<kg.apc.jmeter.samplers.DummySampler guiclass="kg.apc.jmeter.samplers.DummySamplerGui" testclass="kg.apc.jmeter.samplers.DummySampler" testname="jp@gc - Dummy Sampler Slow" enabled="true">
<boolProp name="WAITING">true</boolProp>
<boolProp name="SUCCESFULL">true</boolProp>
<stringProp name="RESPONSE_CODE">200</stringProp>
<stringProp name="RESPONSE_MESSAGE">OK</stringProp>
<stringProp name="RESPONSE_DATA">Degraded slow response</stringProp>
<stringProp name="RESPONSE_TIME">500</stringProp>
<stringProp name="LATENCY">200</stringProp>
<stringProp name="CONNECT">10</stringProp>
<stringProp name="URL">http://dummy</stringProp>
<stringProp name="RESULT_CLASS">org.apache.jmeter.samplers.SampleResult</stringProp>
<stringProp name="REQUEST_DATA"></stringProp>
</kg.apc.jmeter.samplers.DummySampler>
<hashTree/>
</hashTree>
</hashTree>
</hashTree>
</jmeterTestPlan>
Loading
Loading