diff --git a/README.md b/README.md index b4a2f96fb8..d59d279189 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ def test_matches_builtin(ls): assert sorted(ls) == my_sort(ls) ``` -This randomized testing can catch bugs and edge cases that you didn't think of and wouldn't have found. In addition, when Hypothesis does find a bug, it doesn't just report any failing example — it reports the simplest possible one. This makes property-based tests a powerful tool for debugging, as well as testing. +This randomized testing can catch bugs and edge cases that you didn't think of and wouldn't have found. In addition, when Hypothesis does find a bug, it doesn't just report any failing test case — it reports the simplest possible one. This makes property-based tests a powerful tool for debugging, as well as testing. For instance, @@ -30,10 +30,10 @@ def my_sort(ls): return sorted(set(ls)) ``` -fails with the simplest possible failing example: +fails with the simplest possible failing test case: ``` -Falsifying example: test_matches_builtin(ls=[0, 0]) +Failing test case: test_matches_builtin(ls=[0, 0]) ``` ### Installation diff --git a/hypothesis/RELEASE.rst b/hypothesis/RELEASE.rst new file mode 100644 index 0000000000..3aa5283c57 --- /dev/null +++ b/hypothesis/RELEASE.rst @@ -0,0 +1,7 @@ +RELEASE_TYPE: minor + +Hypothesis has historically referred to the single execution of a test function as an "example". However, we have in recent years tended to call a single execution a "test case" instead, which we think is a more precise and less overloaded term. + +This release updates user-facing documentation, log messages, and error messages to use the "test case" terminology. + +This is not a breaking change, as we have intentionally not changed any code APIs. For example, |settings.max_examples| has not been changed. diff --git a/hypothesis/docs/_static/better-signatures.css b/hypothesis/docs/_static/better-signatures.css deleted file mode 100644 index 15f24c51d5..0000000000 --- a/hypothesis/docs/_static/better-signatures.css +++ /dev/null @@ -1,13 +0,0 @@ -/* dl gets used both for defining each top-level `.. autofunc` on a page (where we want vertical margsin) - and is wrapped around multiline signatures (where we don't). - If a dl is being used inside a .sig, that's a multiline signature; remove its margins. */ -.sig > dl { - margin-block-start: 0rem; - margin-block-end: 0rem; -} - -/* with thanks to https://github.com/pradyunsg/furo/discussions/749 */ -.sig:not(.sig-inline) { - padding-left: 0.5em; - text-indent: 0; -} diff --git a/hypothesis/docs/_static/dark-fix.css b/hypothesis/docs/_static/dark-fix.css deleted file mode 100644 index 213b76761a..0000000000 --- a/hypothesis/docs/_static/dark-fix.css +++ /dev/null @@ -1,28 +0,0 @@ -/* -See https://github.com/HypothesisWorks/hypothesis/issues/4588 and -https://github.com/pradyunsg/furo/discussions/909. Once this is fixed in furo, -we can remove this. -*/ - -body[data-theme="dark"] .tooltip .tooltip-content { - background-color: var(--color-background-primary); -} - -body[data-theme="dark"] .tooltip .arrow { - background: var(--color-background-primary); -} - -/* -Furo also renders dark when the theme is "auto" (the default) and the OS -prefers dark, via this same media query. Cover that case too; see -https://github.com/HypothesisWorks/hypothesis/issues/4734. -*/ -@media (prefers-color-scheme: dark) { - body:not([data-theme="light"]) .tooltip .tooltip-content { - background-color: var(--color-background-primary); - } - - body:not([data-theme="light"]) .tooltip .arrow { - background: var(--color-background-primary); - } -} diff --git a/hypothesis/docs/_static/no-scroll.css b/hypothesis/docs/_static/no-scroll.css deleted file mode 100644 index 89be21a50b..0000000000 --- a/hypothesis/docs/_static/no-scroll.css +++ /dev/null @@ -1,5 +0,0 @@ -/* disable autoscroll-to-target behavior - https://github.com/pradyunsg/furo/discussions/384#discussioncomment-2249243 */ -html { - scroll-behavior: auto; -} diff --git a/hypothesis/docs/_static/styles.css b/hypothesis/docs/_static/styles.css new file mode 100644 index 0000000000..e8031efd3c --- /dev/null +++ b/hypothesis/docs/_static/styles.css @@ -0,0 +1,71 @@ +/* dl gets used both for defining each top-level `.. autofunc` on a page (where we want vertical margsin) + and is wrapped around multiline signatures (where we don't). + If a dl is being used inside a .sig, that's a multiline signature; remove its margins. */ +.sig > dl { + margin-block-start: 0rem; + margin-block-end: 0rem; +} + +/* with thanks to https://github.com/pradyunsg/furo/discussions/749 */ +.sig:not(.sig-inline) { + padding-left: 0.5em; + text-indent: 0; +} + +/* override table width restrictions */ +/* thanks to https://github.com/readthedocs/sphinx_rtd_theme/issues/117#issuecomment-153083280 */ +@media screen and (min-width: 767px) { + + .wy-table-responsive table td { + /* !important prevents the common CSS stylesheets from + overriding this as on RTD they are loaded after this stylesheet */ + white-space: normal !important; + } + + .wy-table-responsive { + overflow: visible !important; + } + +} + +/* disable autoscroll-to-target behavior + https://github.com/pradyunsg/furo/discussions/384#discussioncomment-2249243 */ +html { + scroll-behavior: auto; +} + +/* +See https://github.com/HypothesisWorks/hypothesis/issues/4588 and +https://github.com/pradyunsg/furo/discussions/909. Once this is fixed in furo, +we can remove this. +*/ + +body[data-theme="dark"] .tooltip .tooltip-content { + background-color: var(--color-background-primary); +} + +body[data-theme="dark"] .tooltip .arrow { + background: var(--color-background-primary); +} + +/* +Furo also renders dark when the theme is "auto" (the default) and the OS +prefers dark, via this same media query. Cover that case too; see +https://github.com/HypothesisWorks/hypothesis/issues/4734. +*/ +@media (prefers-color-scheme: dark) { + body:not([data-theme="light"]) .tooltip .tooltip-content { + background-color: var(--color-background-primary); + } + + body:not([data-theme="light"]) .tooltip .arrow { + background: var(--color-background-primary); + } +} + +a:has(> .std-term), +a:has(> .std-term):hover { + color: inherit; + text-decoration-color: currentColor; + text-decoration-style: dotted; +} diff --git a/hypothesis/docs/_static/wrap-in-tables.css b/hypothesis/docs/_static/wrap-in-tables.css deleted file mode 100644 index 3ff01dda29..0000000000 --- a/hypothesis/docs/_static/wrap-in-tables.css +++ /dev/null @@ -1,15 +0,0 @@ -/* override table width restrictions */ -/* thanks to https://github.com/readthedocs/sphinx_rtd_theme/issues/117#issuecomment-153083280 */ -@media screen and (min-width: 767px) { - - .wy-table-responsive table td { - /* !important prevents the common CSS stylesheets from - overriding this as on RTD they are loaded after this stylesheet */ - white-space: normal !important; - } - - .wy-table-responsive { - overflow: visible !important; - } - -} diff --git a/hypothesis/docs/compatibility.rst b/hypothesis/docs/compatibility.rst index 9efd343a51..282b67a2f0 100644 --- a/hypothesis/docs/compatibility.rst +++ b/hypothesis/docs/compatibility.rst @@ -155,4 +155,4 @@ We ship type hints with Hypothesis itself. Though we always try to minimize brea We may also find more precise ways to describe the type of various interfaces, or change their type and runtime behaviour together in a way which is otherwise backwards-compatible. -There are known issues with inferring the type of examples generated by |st.deferred|, |st.recursive|, |st.one_of|, |st.dictionaries|, and |st.fixed_dictionaries|. We're following proposed updates to Python's typing standards, but unfortunately the long-standing interfaces of these strategies cannot (yet) be statically typechecked. +There are known issues with inferring the type of values generated by |st.deferred|, |st.recursive|, |st.one_of|, |st.dictionaries|, and |st.fixed_dictionaries|. We're following proposed updates to Python's typing standards, but unfortunately the long-standing interfaces of these strategies cannot (yet) be statically typechecked. diff --git a/hypothesis/docs/conf.py b/hypothesis/docs/conf.py index c316cc768a..e7f838acdd 100644 --- a/hypothesis/docs/conf.py +++ b/hypothesis/docs/conf.py @@ -221,12 +221,7 @@ def process_signature(app, what, name, obj, options, signature, return_annotatio # remove "Hypothesis documentation" from just below logo on the sidebar html_theme_options = {"sidebar_hide_name": True} html_static_path = ["_static"] -html_css_files = [ - "better-signatures.css", - "wrap-in-tables.css", - "no-scroll.css", - "dark-fix.css", -] +html_css_files = ["styles.css"] htmlhelp_basename = "Hypothesisdoc" html_favicon = "../../brand/favicon.ico" html_logo = "../../brand/dragonfly-rainbow-150w.svg" diff --git a/hypothesis/docs/explanation/domain.rst b/hypothesis/docs/explanation/domain.rst index d4a11b7324..f89c7c6514 100644 --- a/hypothesis/docs/explanation/domain.rst +++ b/hypothesis/docs/explanation/domain.rst @@ -40,10 +40,10 @@ An exact answer depends on both the strategy or strategies for the tests, and th Hypothesis' default configuration uses a distribution which is tuned to maximize the chance of finding bugs, in as few executions as possible. We explicitly *don't* aim for a uniform distribution, nor for a 'realistic' distribution of inputs; Hypothesis' goal is to search the domain for a failing input as efficiently as possible. -The test case distribution remains an active area of research and development, and we change it whenever we think that would be a net improvement for users. Today, Hypothesis' default distribution is shaped by a wide variety of techniques and heuristics: +The |test case| distribution remains an active area of research and development, and we change it whenever we think that would be a net improvement for users. Today, Hypothesis' default distribution is shaped by a wide variety of techniques and heuristics: * some are statically designed into strategies - for example, |st.integers| upweights range endpoints, and samples from a mixed distribution over integer bit-widths. -* some are dynamic features of the engine - like replaying prior examples with subsections of the input 'cloned' or otherwise altered, for bugs which trigger only when different fields have the same value (which is otherwise exponentially unlikely). +* some are dynamic features of the engine - like replaying prior test cases with subsections of the input 'cloned' or otherwise altered, for bugs which trigger only when different fields have the same value (which is otherwise exponentially unlikely). * some vary depending on the code under test - we collect interesting-looking constants from imported source files as seeds for test cases. * `swarm testing `__ adds further randomization when choosing which rules to execute in stateful testing. diff --git a/hypothesis/docs/explanation/example-count.rst b/hypothesis/docs/explanation/example-count.rst index e19f15228e..e0636d5a60 100644 --- a/hypothesis/docs/explanation/example-count.rst +++ b/hypothesis/docs/explanation/example-count.rst @@ -4,19 +4,19 @@ How many times will Hypothesis run my test? This is a trickier question than you might expect. The short answer is "exactly |max_examples| times", with the following exceptions: - Less than |max_examples| times, if Hypothesis exhausts the search space early. -- More than |max_examples| times, if Hypothesis retries some examples because either: +- More than |max_examples| times, if Hypothesis retries some |test cases| because either: - They failed an |assume| or |.filter| condition, or - They were too large to continue generating. -- Either less or more than |max_examples| times, if Hypothesis finds a failing example. +- Either less or more than |max_examples| times, if Hypothesis finds a |failing test case|. Read on for details. Search space exhaustion ----------------------- -If Hypothesis detects that there are no more examples left to try, it may stop generating examples before it hits |max_examples|. For example: +If Hypothesis detects that there are no more test cases left to try, it may stop generating test cases before it hits |max_examples|. For example: .. code-block:: python @@ -45,7 +45,7 @@ The search space tracking in Hypothesis is good, but not perfect. We treat this |assume| and |.filter| ---------------------- -If an example fails to satisfy an |assume| or |.filter| condition, Hypothesis will retry generating that example and will not count it towards the |max_examples| limit. For instance: +If a test case fails to satisfy an |assume| or |.filter| condition, Hypothesis will retry generating that test case and will not count it towards the |max_examples| limit. For instance: .. code-block:: python @@ -55,25 +55,25 @@ If an example fails to satisfy an |assume| or |.filter| condition, Hypothesis wi def test_function(n): assume(n % 2 == 0) -will run roughly 200 times, since half of the examples are discarded from the |assume|. +will run roughly 200 times, since half of the test cases are discarded from the |assume|. -Note that while failing an |assume| triggers an immediate retry of the entire example, Hypothesis will try several times in the same example to satisfy a |.filter| condition. This makes expressing the same condition using |.filter| more efficient than |assume|. +Note that while failing an |assume| triggers an immediate retry of the entire test case, Hypothesis will try several times in the same test case to satisfy a |.filter| condition. This makes expressing the same condition using |.filter| more efficient than |assume|. Also note that even if your code does not explicitly use |assume| or |.filter|, a builtin strategy may still use them and cause retries. We try to directly satisfy conditions where possible instead of relying on rejection sampling, so this should be relatively uncommon. -Examples which are too large ----------------------------- +Test cases which are too large +------------------------------ -For performance reasons, Hypothesis places an internal limit on the size of a single example. If an example exceeds this size limit, we will retry generating it and will not count it towards the |max_examples| limit. (And if we see too many of these large examples, we will raise |HealthCheck.data_too_large|, unless suppressed with |settings.suppress_health_check|). +For performance reasons, Hypothesis places an internal limit on the size of a single test case. If a test case exceeds this size limit, we will retry generating it and will not count it towards the |max_examples| limit. (And if we see too many of these large test cases, we will raise |HealthCheck.data_too_large|, unless suppressed with |settings.suppress_health_check|). The specific value of this size limit is an undocumented implementation detail. The majority of Hypothesis tests do not come close to hitting it. -Failing examples ----------------- +Failing test cases +------------------ -If Hypothesis finds a failing example, it stops generation early, and may call the test function additional times during the |Phase.shrink| and |Phase.explain| phases. Sometimes, Hypothesis determines that the initial failing example was already as simple as possible, in which case |Phase.shrink| will not result in additional test executions (but |Phase.explain| might). +If Hypothesis finds a failing test case, it stops generation early, and may call the test function additional times during the |Phase.shrink| and |Phase.explain| phases. Sometimes, Hypothesis determines that the initial failing test case was already as simple as possible, in which case |Phase.shrink| will not result in additional test executions (but |Phase.explain| might). -Regardless of whether Hypothesis runs the test during the shrinking and explain phases, it will always run the minimal failing example one additional time to check for flakiness. For instance, the following trivial test runs with ``n=0`` *twice*, even though it only uses the |Phase.generate| phase: +Regardless of whether Hypothesis runs the test during the shrinking and explain phases, it will always run the minimal failing test case one additional time to check for flakiness. For instance, the following trivial test runs with ``n=0`` *twice*, even though it only uses the |Phase.generate| phase: .. code-block:: python diff --git a/hypothesis/docs/glossary.rst b/hypothesis/docs/glossary.rst new file mode 100644 index 0000000000..64da3a1834 --- /dev/null +++ b/hypothesis/docs/glossary.rst @@ -0,0 +1,86 @@ +Glossary +======== + +User glossary +------------- + +Terms that are part of our public API. + +.. glossary:: + + explicit example + A |test case| from |@example|. + + .. code-block:: python + + @example(42) + def f(n): + pass + + Here, ``42`` is an explicit example. + + Explicit examples are always run, in the |Phase.explicit| phase. Unlike Hypothesis-generated test cases, Hypothesis does not shrink explicit examples. + + failing test case + A |test case| which causes the test to fail, usually by causing an exception to be raised. + + minimal failing test case + The |failing test case| which has been fully shrunk (minimized) by Hypothesis. Hypothesis reports only the minimal failing test case to the user at the end of the test. + + .. code-block:: python + + @given(st.integers()) + def f(n): + print("called with", n) + assert n < 10 + + .. code-block:: none + + called with 0 + called with 921 + called with 212 + ... + called with 10 + ... + Failing test case: f( + n=10, + ) + + Here, each of ``921``, ``212``, and ``10`` are failing test cases. ``10`` is the minimal failing test case. + + test case + The Hypothesis-generated input to a test function. + + .. code-block:: python + + @given(st.integers()) + def f(n): + print("called with", n) + assume(n >= 0) + + .. code-block:: none + + called with 0 + called with 18588 + called with 672780074 + called with -32616 + ... + + Here, the first four test cases are ``0``, ``18588``, ``672780074``, and ``-32616``. + + "Test case" may also refer to the resulting execution of an input in a test function. Above, we might say that "the test case ``-32616`` failed the |assume|", referring to its entire execution. + + +Developer glossary +------------------ + +Terms that are part of our developer API. The developer API is intended for advanced users, researchers, and developers building on top of Hypothesis. + +.. glossary:: + + choice sequence + The underlying sequence of primitive values corresponding to a :term:`test case`. Conceptually, a choice sequence is the sequence of random choices made in the course of generating a value in a strategy. Each test case is represented internally as a choice sequence. + + The exact representation is an implementation detail and depends on the strategy. For example, the value ``True`` from |st.booleans| is represented by the choice sequence ``[True]``, while the value ``[2, 42]`` from ``st.lists(st.integers())`` is represented by the choice sequence ``[True, 2, True, 42, False]``. + + Currently, a choice sequence consists of booleans, integers, floats, strings, and bytes. diff --git a/hypothesis/docs/how-to/external-fuzzers.rst b/hypothesis/docs/how-to/external-fuzzers.rst index 7832c7c2fe..937cce6dce 100644 --- a/hypothesis/docs/how-to/external-fuzzers.rst +++ b/hypothesis/docs/how-to/external-fuzzers.rst @@ -8,7 +8,7 @@ Sometimes you might want to point a traditional fuzzer like `python-afl `_. This page is about writing traditional 'fuzz harnesses' with an external fuzzer, using parts of Hypothesis. -In order to support this workflow, Hypothesis exposes the |fuzz_one_input| method. |fuzz_one_input| takes a bytestring, parses it into a test case, and executes the corresponding test once. This means you can treat each of your Hypothesis tests as a traditional fuzz target, by pointing the fuzzer at |fuzz_one_input|. +In order to support this workflow, Hypothesis exposes the |fuzz_one_input| method. |fuzz_one_input| takes a bytestring, parses it into a |test case|, and executes the corresponding test once. This means you can treat each of your Hypothesis tests as a traditional fuzz target, by pointing the fuzzer at |fuzz_one_input|. For example: diff --git a/hypothesis/docs/how-to/suppress-healthchecks.rst b/hypothesis/docs/how-to/suppress-healthchecks.rst index 157a2072ee..489f8c3967 100644 --- a/hypothesis/docs/how-to/suppress-healthchecks.rst +++ b/hypothesis/docs/how-to/suppress-healthchecks.rst @@ -1,7 +1,7 @@ Suppress a health check everywhere ================================== -Hypothesis sometimes raises a |HealthCheck| to indicate that your test may be less effective than you expect, slower than you expect, unlikely to generate effective examples, or otherwise has silently degraded performance. +Hypothesis sometimes raises a |HealthCheck| to indicate that your test may be less effective than you expect, slower than you expect, unlikely to generate effective |test cases|, or otherwise has silently degraded performance. While |HealthCheck| can be useful to proactively identify issues, you may not care about certain classes of them. If you want to disable a |HealthCheck| everywhere, you can register and load a settings profile with |settings.register_profile| and |settings.load_profile|. Place the following code in any file which is loaded before running your tests (or in ``conftest.py``, if using pytest): diff --git a/hypothesis/docs/how-to/type-strategies.rst b/hypothesis/docs/how-to/type-strategies.rst index 2c24dc19c3..9ea38bdc0e 100644 --- a/hypothesis/docs/how-to/type-strategies.rst +++ b/hypothesis/docs/how-to/type-strategies.rst @@ -13,7 +13,7 @@ Hypothesis provides type hints for all strategies and functions which return a s reveal_type(st.lists(st.integers())) # SearchStrategy[list[int]] -|SearchStrategy| is the type of a strategy. It is parametrized by the type of the example it generates. You can use it to write type hints for your functions which return a strategy: +|SearchStrategy| is the type of a strategy. It is parametrized by the type of the values it generates. You can use it to write type hints for your functions which return a strategy: .. code-block:: python diff --git a/hypothesis/docs/index.rst b/hypothesis/docs/index.rst index afc4f74f6a..11c9d67063 100644 --- a/hypothesis/docs/index.rst +++ b/hypothesis/docs/index.rst @@ -102,6 +102,7 @@ Technical API reference. reference/index stateful Extras + glossary changelog .. toctree:: diff --git a/hypothesis/docs/prolog.rst b/hypothesis/docs/prolog.rst index 3de146e626..6d0aec992f 100644 --- a/hypothesis/docs/prolog.rst +++ b/hypothesis/docs/prolog.rst @@ -216,4 +216,16 @@ .. |alternative backend| replace:: :ref:`alternative backend ` .. |alternative backends| replace:: :ref:`alternative backends ` +.. |test case| replace:: :term:`test case` +.. |test cases| replace:: :term:`test cases ` +.. |explicit example| replace:: :term:`explicit example` +.. |explicit examples| replace:: :term:`explicit examples ` +.. |Explicit examples| replace:: :term:`Explicit examples ` +.. |failing test case| replace:: :term:`failing test case` +.. |failing test cases| replace:: :term:`failing test cases ` +.. |minimal failing test case| replace:: :term:`minimal failing test case` +.. |minimal failing test cases| replace:: :term:`minimal failing test cases ` +.. |choice sequence| replace:: :term:`choice sequence` +.. |choice sequences| replace:: :term:`choice sequences ` + .. |pytest.mark.parametrize| replace:: :ref:`pytest.mark.parametrize ` diff --git a/hypothesis/docs/quickstart.rst b/hypothesis/docs/quickstart.rst index 43219d413b..0bb7f996b0 100644 --- a/hypothesis/docs/quickstart.rst +++ b/hypothesis/docs/quickstart.rst @@ -70,7 +70,7 @@ This test will clearly fail, which can be confirmed by running ``pytest example. def test_integers(n): > assert n < 50 E assert 50 < 50 - E Falsifying example: test_integers( + E Failing test case: test_integers( E n=50, E ) @@ -111,7 +111,7 @@ Sometimes, you need to remove invalid cases from your test. The best way to do t def test_integers(n): assert n % 2 == 0 -For more complicated conditions, you can use |assume|, which tells Hypothesis to discard any test case with a false-y argument: +For more complicated conditions, you can use |assume|, which tells Hypothesis to discard any |test case| with a false-y argument: .. code-block:: python diff --git a/hypothesis/docs/reference/api.rst b/hypothesis/docs/reference/api.rst index c70b0458d6..0c1a98bfdc 100644 --- a/hypothesis/docs/reference/api.rst +++ b/hypothesis/docs/reference/api.rst @@ -95,7 +95,7 @@ Reproducing inputs Control ------- -Functions that can be called from anywhere inside a test, to either modify how Hypothesis treats the current test case, or to give Hypothesis more information about the current test case. +Functions that can be called from anywhere inside a test, to either modify how Hypothesis treats the current |test case|, or to give Hypothesis more information about the current test case. .. autofunction:: hypothesis.assume .. autofunction:: hypothesis.note @@ -121,7 +121,7 @@ For instance, in the latter case, you would see output like: - during generate phase (0.09 seconds): - Typical runtimes: < 1ms, ~ 59% in data generation - - 100 passing examples, 0 failing examples, 32 invalid examples + - 100 passing test cases, 0 failing test cases, 32 invalid test cases - Events: * 54.55%, Retried draw from integers().filter(lambda x: x % 2 == 0) to satisfy filter * 31.06%, i mod 3 = 2 @@ -262,7 +262,7 @@ Using it is quite straightforward: All you need to do is subclass :class:`~hypothesis.extra.django.LiveServerTestCase` or :class:`~hypothesis.extra.django.StaticLiveServerTestCase` and you can use |@given| as normal, -and the transactions will be per example +and the transactions will be per test case rather than per test function as they would be if you used |@given| with a normal django test suite (this is important because your test function will be called multiple times and you don't want them to interfere with each other). Test cases @@ -280,7 +280,7 @@ Because Hypothesis runs this in a loop, the performance problems :class:`django: are significantly exacerbated and your tests will be really slow. If you are using :class:`~hypothesis.extra.django.TransactionTestCase`, you may need to use ``@settings(suppress_health_check=[HealthCheck.too_slow])`` -to avoid a |HealthCheck| error due to slow example generation. +to avoid a |HealthCheck| error due to slow test case generation. Having set up a test class, you can now pass |@given| a strategy for Django models with |django.from_model|. @@ -309,7 +309,7 @@ to set the valid range for this field (or used a :class:`~django:django.db.models.PositiveSmallIntegerField`, which would only need a maximum value validator). -If you *do* have validators attached, Hypothesis will only generate examples +If you *do* have validators attached, Hypothesis will only generate test cases that pass validation. Sometimes that will mean that we fail a :class:`~hypothesis.HealthCheck` because of the filtering, so let's explicitly pass a strategy to skip validation at the strategy level: @@ -451,9 +451,9 @@ External fuzzers Custom function execution ------------------------- -Hypothesis provides you with a hook that lets you control how it runs examples. +Hypothesis provides you with a hook that lets you control how it runs test cases. -This lets you do things like set up and tear down around each example, run examples in a subprocess, transform coroutine tests into normal tests, etc. For example, :class:`~hypothesis.extra.django.TransactionTestCase` in the Django extra runs each example in a separate database transaction. +This lets you do things like set up and tear down around each test case, run test cases in a subprocess, transform coroutine tests into normal tests, etc. For example, :class:`~hypothesis.extra.django.TransactionTestCase` in the Django extra runs each test case in a separate database transaction. The way this works is by introducing the concept of an executor. An executor is essentially a function that takes a block of code and run it. The default executor is: diff --git a/hypothesis/docs/reference/integrations.rst b/hypothesis/docs/reference/integrations.rst index 0cc847a071..83911de5a4 100644 --- a/hypothesis/docs/reference/integrations.rst +++ b/hypothesis/docs/reference/integrations.rst @@ -96,11 +96,11 @@ is to let you answer questions you didn't think of in advance. In slogan form, *Debugging should be a data analysis problem.* -By default, Hypothesis only reports the minimal failing example... but sometimes you might -want to know something about *all* the examples. Printing them to the terminal by increasing +By default, Hypothesis only reports the |minimal failing test case|... but sometimes you might +want to know something about *all* the test cases. Printing them to the terminal by increasing |Verbosity| might be nice, but isn't always enough. This feature gives you an analysis-ready dataframe with useful columns and one row -per test case, with columns from arguments to code coverage to pass/fail status. +per |test case|, with columns from arguments to code coverage to pass/fail status. This is deliberately a much lighter-weight and task-specific system than e.g. `OpenTelemetry `__. It's also less detailed than time-travel @@ -233,10 +233,10 @@ You would see: - during generate phase (0.06 seconds): - Typical runtimes: < 1ms, ~ 47% in data generation - - 100 passing examples, 0 failing examples, 0 invalid examples + - 100 passing test cases, 0 failing test cases, 0 invalid test cases - Stopped because settings.max_examples=100 -The final "Stopped because" line tells you why Hypothesis stopped generating new examples. This is typically because we hit |max_examples|, but occasionally because we exhausted the search space or because shrinking was taking a very long time. This can be useful for understanding the behaviour of your tests. +The final "Stopped because" line tells you why Hypothesis stopped generating new test cases. This is typically because we hit |max_examples|, but occasionally because we exhausted the search space or because shrinking was taking a very long time. This can be useful for understanding the behaviour of your tests. In some cases (such as filtered and recursive strategies) you will see events mentioned which describe some aspect of the data generation: @@ -256,7 +256,7 @@ You would see something like: - during generate phase (0.08 seconds): - Typical runtimes: < 1ms, ~ 57% in data generation - - 100 passing examples, 0 failing examples, 12 invalid examples + - 100 passing test cases, 0 failing test cases, 12 invalid test cases - Events: * 51.79%, Retried draw from integers().filter(lambda x: x % 2 == 0) to satisfy filter * 10.71%, Aborted test because unable to satisfy integers().filter(lambda x: x % 2 == 0) diff --git a/hypothesis/docs/stateful.rst b/hypothesis/docs/stateful.rst index cdadfac868..acbb908389 100644 --- a/hypothesis/docs/stateful.rst +++ b/hypothesis/docs/stateful.rst @@ -117,7 +117,7 @@ You can control the detailed behaviour with a settings object on the TestCase (t max_examples=50, stateful_step_count=100 ) -Which doubles the number of steps each program runs and halves the number of test cases that will be run. +Which doubles the number of steps each program runs and halves the number of |test cases| that will be run. Rules ----- diff --git a/hypothesis/docs/tutorial/adapting-strategies.rst b/hypothesis/docs/tutorial/adapting-strategies.rst index 0343c84a9d..ac743daa9c 100644 --- a/hypothesis/docs/tutorial/adapting-strategies.rst +++ b/hypothesis/docs/tutorial/adapting-strategies.rst @@ -13,7 +13,7 @@ Sometimes you want to apply a simple transformation to a strategy. For instance, >>> lists(integers()).map(sorted).example() [-25527, -24245, -93, -70, -7, 0, 39, 65, 112, 6189, 19469, 32526, 1566924430] -In general, ``strategy.map(f)`` returns a new strategy which transforms all the examples generated by ``strategy`` by calling ``f`` on them. +In general, ``strategy.map(f)`` returns a new strategy which transforms all the values generated by ``strategy`` by calling ``f`` on them. Filtering strategy inputs ------------------------- @@ -54,7 +54,7 @@ Calling |.filter| on a strategy creates a new strategy with that filter applied Assuming away test cases ------------------------ -|.filter| lets you filter test inputs from a single strategy. Hypothesis also provides an |assume| function for when you need to filter an entire test case, based on an arbitrary condition. +|.filter| lets you filter test inputs from a single strategy. Hypothesis also provides an |assume| function for when you need to filter an entire |test case|, based on an arbitrary condition. The |assume| function skips test cases where some condition evaluates to ``False``. You can use it anywhere in your test. We could have expressed our |.filter| example above using |assume| as well: diff --git a/hypothesis/docs/tutorial/adding-notes.rst b/hypothesis/docs/tutorial/adding-notes.rst index 004815455f..279ce689eb 100644 --- a/hypothesis/docs/tutorial/adding-notes.rst +++ b/hypothesis/docs/tutorial/adding-notes.rst @@ -5,7 +5,7 @@ When a test fails, Hypothesis will normally print output that looks like this: .. code:: - Falsifying example: test_a_thing(x=1, y="foo") + Failing test case: test_a_thing(x=1, y="foo") Sometimes you want to add some additional information to a failure, such as the output of some intermediate step in your test. The |note| function lets you do this: @@ -24,12 +24,12 @@ Sometimes you want to add some additional information to a failure, such as the ... except AssertionError: ... print("ls != ls2") ... - Falsifying example: test_shuffle_is_noop(ls=[0, 1], r=RandomWithSeed(1)) + Failing test case: test_shuffle_is_noop(ls=[0, 1], r=RandomWithSeed(1)) Shuffle: [1, 0] ls != ls2 -|note| is like a print statement that gets attached to the falsifying example reported by Hypothesis. It's also reported by :ref:`observability `, and shown for all examples (if |settings.verbosity| is set to |Verbosity.verbose| or higher). +|note| is like a print statement that gets attached to the |minimal failing test case| reported by Hypothesis. It's also reported by :ref:`observability `, and shown for all test cases (if |settings.verbosity| is set to |Verbosity.verbose| or higher). .. note:: - |event| is a similar function which tells Hypothesis to count the number of test cases which reported each distinct value you pass, for inclusion in :ref:`test statistics ` and :ref:`observability reports `. + |event| is a similar function which tells Hypothesis to count the number of |test cases| which reported each distinct value you pass, for inclusion in :ref:`test statistics ` and :ref:`observability reports `. diff --git a/hypothesis/docs/tutorial/custom-strategies.rst b/hypothesis/docs/tutorial/custom-strategies.rst index f36cfe3761..80afa3b285 100644 --- a/hypothesis/docs/tutorial/custom-strategies.rst +++ b/hypothesis/docs/tutorial/custom-strategies.rst @@ -132,7 +132,7 @@ When using |st.composite|, you have to finish generating the entire input before .. note:: - The downside of this power is that |st.data| is incompatible |@example|, and that Hypothesis cannot print a nice representation of values generated from |st.data| when reporting failing examples, because the draws are spread out. Where possible, prefer |st.composite| to |st.data|. + The downside of this power is that |st.data| is incompatible |@example|, and that Hypothesis cannot print a nice representation of values generated from |st.data| when reporting |failing test cases|, because the draws are spread out. Where possible, prefer |st.composite| to |st.data|. For instance, here's how we would write our earlier |st.composite| example using |st.data|: diff --git a/hypothesis/docs/tutorial/flaky.rst b/hypothesis/docs/tutorial/flaky.rst index 488478e8a6..c569a32506 100644 --- a/hypothesis/docs/tutorial/flaky.rst +++ b/hypothesis/docs/tutorial/flaky.rst @@ -60,8 +60,8 @@ As a result, running ``test_fails_flakily()`` will raise |FlakyFailure|. |FlakyF .. code-block:: none + Exception Group Traceback (most recent call last): - | hypothesis.errors.FlakyFailure: Hypothesis test_fails_flakily(n=0) produces unreliable results: Falsified on the first call but did not on a subsequent one (1 sub-exception) - | Falsifying example: test_fails_flakily( + | hypothesis.errors.FlakyFailure: Hypothesis test_fails_flakily(n=0) produces unreliable results: Failed on the first call but did not on a subsequent one (1 sub-exception) + | Failing test case: test_fails_flakily( | n=0, | ) | Failed to reproduce exception. Expected: @@ -86,7 +86,7 @@ Flaky strategy definition Each strategy must 'do the same thing' (again, as seen by Hypothesis) if we replay a previously-seen input. Failing to do so is a more subtle but equally serious form of flakiness, which leaves us unable to shrink to a minimal failing input, or even reliably report the failure in future runs. -One easy way for this to occur is if a strategy depends on external state. For example, this strategy filters out previously-generated integers, including those seen in any previous test case: +One easy way for this to occur is if a strategy depends on external state. For example, this strategy filters out previously-generated integers, including those seen in any previous |test case|: .. code-block:: python diff --git a/hypothesis/docs/tutorial/replaying-failures.rst b/hypothesis/docs/tutorial/replaying-failures.rst index 9126e8bc20..a6ae1e8325 100644 --- a/hypothesis/docs/tutorial/replaying-failures.rst +++ b/hypothesis/docs/tutorial/replaying-failures.rst @@ -38,7 +38,7 @@ You can disable the database by passing ``database=None`` to |@settings|: Using |@example| to run a specific input ---------------------------------------- -If you want Hypothesis to always run a specific input, you can use |@example|. |@example| adds an explicit input which Hypothesis will run every time, in addition to the randomly generated examples. You can think of |@example| as combining unit-testing with property-based testing. +If you want Hypothesis to always run a specific input, you can use |@example|. |@example| adds an explicit input which Hypothesis will run every time, in addition to the randomly generated |test cases|. You can think of |@example| as combining unit-testing with property-based testing. For instance, suppose we write a test using |st.integers|, but want to make sure we try a few special prime numbers every time we run the test. We can add these inputs with |@example|: @@ -53,7 +53,7 @@ For instance, suppose we write a test using |st.integers|, but want to make sure test_integers() -Hypothesis runs all explicit examples first, in the |Phase.explicit| phase, before generating additional random examples in the |Phase.generate| phase. +Hypothesis runs all |explicit examples| first, in the |Phase.explicit| phase, before generating additional random test cases in the |Phase.generate| phase. Inputs from |@example| do not shrink ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -67,7 +67,7 @@ Note that unlike examples generated by Hypothesis, examples provided using |@exa def test_something_with_integers(n): assert n < 100 -Hypothesis will print ``Falsifying explicit example: test_something_with_integers(n=131071)``, instead of shrinking to ``n=100``. +Hypothesis will print ``Failing explicit example: test_something_with_integers(n=131071)``, instead of shrinking to ``n=100``. Prefer |@example| over the database for correctness --------------------------------------------------- @@ -93,10 +93,10 @@ If |settings.print_blob| is set to ``True`` (the default in the ``ci`` settings >>> test() ... - Falsifying example: test( + Failing test case: test( f=nan, ) - You can reproduce this example by temporarily adding @reproduce_failure('6.131.23', b'ACh/+AAAAAAAAA==') as a decorator on your test case + You can reproduce this test case by temporarily adding @reproduce_failure('6.131.23', b'ACh/+AAAAAAAAA==') as a decorator on your test function You can add this decorator to your test to reproduce the failure. This can be useful for locally replaying failures found by CI. Note that the binary blob is not stable across Hypothesis versions, so you should not leave this decorator on your tests permanently. Use |@example| with an explicit input instead. diff --git a/hypothesis/docs/tutorial/settings.rst b/hypothesis/docs/tutorial/settings.rst index 146aa6f91a..077f751505 100644 --- a/hypothesis/docs/tutorial/settings.rst +++ b/hypothesis/docs/tutorial/settings.rst @@ -6,7 +6,7 @@ This page discusses how to configure the behavior of a single Hypothesis test, o Configuring a single test ------------------------- -Hypothesis lets you configure the default behavior of a test using the |@settings| decorator. You can use settings to configure how many examples Hypothesis generates, how Hypothesis replays failing examples, and the verbosity level of the test, among others. +Hypothesis lets you configure the default behavior of a test using the |@settings| decorator. You can use settings to configure how many |test cases| Hypothesis generates, how Hypothesis replays |failing test cases|, and the verbosity level of the test, among others. Using |@settings| on a single test looks like this: @@ -21,10 +21,10 @@ Using |@settings| on a single test looks like this: You can put |@settings| either before or after |@given|. Both are equivalent. -Changing the number of examples -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Changing the number of test cases +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you have a test which is very expensive or very cheap to run, you can change the number of examples (inputs) Hypothesis generates with the |max_examples| setting: +If you have a test which is very expensive or very cheap to run, you can change the number of test cases (inputs) Hypothesis generates with the |max_examples| setting: .. code-block:: python @@ -35,7 +35,7 @@ If you have a test which is very expensive or very cheap to run, you can change def test(n): print("prints five times") -The default is 100 examples. +The default is 100 test cases. .. note:: @@ -48,7 +48,7 @@ Other settings options Here are a few of the more commonly used setting values: * |settings.phases| controls which phases of Hypothesis run, like replaying from the database or generating new inputs. -* |settings.database| controls how and if Hypothesis replays failing examples. +* |settings.database| controls how and if Hypothesis replays failing test cases. * |settings.verbosity| can print debug information. * |settings.derandomize| makes Hypothesis deterministic. (`Two kinds of testing `__ discusses when and why you might want that). @@ -60,7 +60,7 @@ Here are a few of the more commonly used setting values: Changing settings across your test suite ---------------------------------------- -In addition to configuring individual test functions with |@settings|, you can configure test behavior across your test suite using a settings profile. This might be useful for creating a development settings profile which runs fewer examples, or a settings profile in CI which connects to a separate database. +In addition to configuring individual test functions with |@settings|, you can configure test behavior across your test suite using a settings profile. This might be useful for creating a development settings profile which runs fewer test cases, or a settings profile in CI which connects to a separate database. To create a settings profile, use |settings.register_profile|: @@ -81,12 +81,12 @@ Note that registering a new profile will not affect tests until it is loaded wit settings.register_profile("fast", max_examples=10) # any tests executed before loading this profile will still use the - # default active profile of 100 examples. + # default active profile of 100 test cases. settings.load_profile("fast") # any tests executed after this point will use the active fast - # profile of 10 examples. + # profile of 10 test cases. There is no limit to the number of settings profiles you can create. Hypothesis creates a profile called ``"default"``, which is active by default. You can also explicitly make it active at any time using ``settings.load_profile("default")``, if for instance you wanted to revert a custom profile you had previously loaded. diff --git a/hypothesis/pyproject.toml b/hypothesis/pyproject.toml index 28df9e4da0..2ba08eef3b 100644 --- a/hypothesis/pyproject.toml +++ b/hypothesis/pyproject.toml @@ -70,7 +70,7 @@ def test_matches_builtin(ls): assert sorted(ls) == my_sort(ls) ``` -This randomized testing can catch bugs and edge cases that you didn't think of and wouldn't have found. In addition, when Hypothesis does find a bug, it doesn't just report any failing example — it reports the simplest possible one. This makes property-based tests a powerful tool for debugging, as well as testing. +This randomized testing can catch bugs and edge cases that you didn't think of and wouldn't have found. In addition, when Hypothesis does find a bug, it doesn't just report any failing test case — it reports the simplest possible one. This makes property-based tests a powerful tool for debugging, as well as testing. For instance, @@ -79,10 +79,10 @@ def my_sort(ls): return sorted(set(ls)) ``` -fails with the simplest possible failing example: +fails with the simplest possible failing test case: ``` -Falsifying example: test_matches_builtin(ls=[0, 0]) +Failing test case: test_matches_builtin(ls=[0, 0]) ``` ### Installation diff --git a/hypothesis/src/_hypothesis_pytestplugin.py b/hypothesis/src/_hypothesis_pytestplugin.py index 6838e3ac76..42109ab436 100644 --- a/hypothesis/src/_hypothesis_pytestplugin.py +++ b/hypothesis/src/_hypothesis_pytestplugin.py @@ -405,7 +405,7 @@ def pytest_terminal_summary(terminalreporter): terminalreporter.write_line(f"observations written to {fname}") if failing_examples: - # This must have been imported already to write the failing examples + # This must have been imported already to write the failing test cases from hypothesis.extra._patching import gc_patches, make_patch, save_patch patch = make_patch(failing_examples) @@ -418,7 +418,7 @@ def pytest_terminal_summary(terminalreporter): if not _WROTE_TO: terminalreporter.section("Hypothesis") terminalreporter.write_line( - f"`git apply {fname}` to add failing examples to your code." + f"`git apply {fname}` to add failing test cases to your code." ) def pytest_collection_modifyitems(items): diff --git a/hypothesis/src/hypothesis/__init__.py b/hypothesis/src/hypothesis/__init__.py index 0efe43985d..42ba7b390d 100644 --- a/hypothesis/src/hypothesis/__init__.py +++ b/hypothesis/src/hypothesis/__init__.py @@ -12,7 +12,7 @@ some source of data. It verifies your code against a wide range of input and minimizes any -failing examples it finds. +failing test cases it finds. """ import _hypothesis_globals diff --git a/hypothesis/src/hypothesis/_settings.py b/hypothesis/src/hypothesis/_settings.py index 0d66eb6336..49f9d70d43 100644 --- a/hypothesis/src/hypothesis/_settings.py +++ b/hypothesis/src/hypothesis/_settings.py @@ -67,13 +67,13 @@ class Verbosity(Enum): quiet = "quiet" """ - Hypothesis will not print any output, not even the final falsifying example. + Hypothesis will not print any output, not even the |minimal failing test case|. """ normal = "normal" """ - Standard verbosity. Hypothesis will print the falsifying example, alongside - any notes made with |note| (only for the falsfying example). + Standard verbosity. Hypothesis will print the |minimal failing test case|, alongside + any notes made with |note| (only for the minimal failing test case). """ verbose = "verbose" @@ -81,7 +81,7 @@ class Verbosity(Enum): Increased verbosity. In addition to everything in |Verbosity.normal|, Hypothesis will: - * Print each test case as it tries it + * Print each |test case| as it tries it * Print any notes made with |note| for each test case * Print each shrinking attempt * Print all explicit failing examples when using |@example|, instead of only @@ -144,27 +144,27 @@ class Phase(Enum): explicit = "explicit" """ - Controls whether explicit examples are run. + Controls whether |explicit examples| are run. """ reuse = "reuse" """ - Controls whether previous examples will be reused. + Controls whether previous test cases will be reused. """ generate = "generate" """ - Controls whether new examples will be generated. + Controls whether new test cases will be generated. """ target = "target" """ - Controls whether examples will be mutated for targeting. + Controls whether test cases will be mutated for targeting. """ shrink = "shrink" """ - Controls whether examples will be shrunk. + Controls whether failing test cases will be shrunk. """ explain = "explain" @@ -172,7 +172,7 @@ class Phase(Enum): Controls whether Hypothesis attempts to explain test failures. The explain phase has two parts, each of which is best-effort - if Hypothesis - can't find a useful explanation, we'll just print the minimal failing example. + can't find a useful explanation, we'll just print the |minimal failing test case|. """ @classmethod @@ -296,7 +296,7 @@ def all(cls) -> list["HealthCheck"]: return list(HealthCheck) data_too_large = "data_too_large" - """Checks if too many examples are aborted for being too large. + """Checks if too many test cases are aborted for being too large. This is measured by the number of random choices that Hypothesis makes in order to generate something, not the size of the generated object. @@ -306,7 +306,7 @@ def all(cls) -> list["HealthCheck"]: """ filter_too_much = "filter_too_much" - """Check for when the test is filtering out too many examples, either + """Check for when the test is filtering out too many test cases, either through use of |assume| or |.filter|, or occasionally for Hypothesis internal reasons.""" @@ -334,16 +334,16 @@ def all(cls) -> list["HealthCheck"]: function_scoped_fixture = "function_scoped_fixture" """Checks if |@given| has been applied to a test with a pytest function-scoped fixture. Function-scoped fixtures run once - for the whole function, not once per example, and this is usually not what + for the whole function, not once per test case, and this is usually not what you want. Because of this limitation, tests that need to set up or reset - state for every example need to do so manually within the test itself, + state for every test case need to do so manually within the test itself, typically using an appropriate context manager. Suppress this health check only in the rare case that you are using a function-scoped fixture that does not need to be reset between individual - examples, but for some reason you cannot use a wider fixture scope + test cases, but for some reason you cannot use a wider fixture scope (e.g. session scope, module scope, class scope). This check requires the :ref:`Hypothesis pytest plugin`, @@ -754,23 +754,33 @@ def __init__( @property def max_examples(self): """ - Once this many satisfying examples have been considered without finding any - counter-example, Hypothesis will stop looking. + Once this many satisfying |test cases| have been considered without finding + any failing test case, Hypothesis will stop looking. + + .. note:: + + Historically, Hypothesis referred to test cases as "examples", including in + this setting name. We now refer to them as test cases throughout our + documentation and codebase. + + This setting is an exception: to avoid ecosystem churn, this setting will + continue to be called ``max_examples``. Conceptually, it is better thought of + as "max test cases". Note that we might call your test function fewer times if we find a bug early or can tell that we've exhausted the search space; or more if we discard some - examples due to use of .filter(), assume(), or a few other things that can + test cases due to use of .filter(), assume(), or a few other things that can prevent the test case from completing successfully. The default value is chosen to suit a workflow where the test will be part of a suite that is regularly executed locally or on a CI server, balancing total running time against the chance of missing a bug. - If you are writing one-off tests, running tens of thousands of examples is + If you are writing one-off tests, running tens of thousands of test cases is quite reasonable as Hypothesis may miss uncommon bugs with default settings. For very complex code, we have observed Hypothesis finding novel bugs after - *several million* examples while testing :pypi:`SymPy `. - If you are running more than 100k examples for a test, consider using our + *several million* test cases while testing :pypi:`SymPy `. + If you are running more than 100k test cases for a test, consider using our :ref:`integration for coverage-guided fuzzing ` - it really shines when given minutes or hours to run. @@ -782,7 +792,7 @@ def max_examples(self): def derandomize(self): """ If True, seed Hypothesis' random number generator using a hash of the test - function, so that every run will test the same set of examples until you + function, so that every run will test the same set of test cases until you update Hypothesis, Python, or the test function. This allows you to `check for regressions and look for bugs @@ -798,8 +808,8 @@ def derandomize(self): @property def database(self): """ - An instance of |ExampleDatabase| that will be used to save examples to - and load previous examples from. + An instance of |ExampleDatabase| that will be used to save failing test + cases to, and load previous failing test cases from. If not set, a |DirectoryBasedExampleDatabase| is created in the current working directory under ``.hypothesis/examples``. If this location is @@ -850,18 +860,18 @@ def verbosity(self): ... def f(x): ... assert not any(x) ... f() - Trying example: [] - Falsifying example: [-1198601713, -67, 116, -29578] - Shrunk example to [-1198601713] - Shrunk example to [-128] - Shrunk example to [32] - Shrunk example to [1] + Test case: [] + Failing test case: [-1198601713, -67, 116, -29578] + Shrunk test case to [-1198601713] + Shrunk test case to [-128] + Shrunk test case to [32] + Shrunk test case to [1] [1] The four levels are |Verbosity.quiet|, |Verbosity.normal|, |Verbosity.verbose|, and |Verbosity.debug|. |Verbosity.normal| is the default. For |Verbosity.quiet|, Hypothesis will not print anything out, - not even the final falsifying example. |Verbosity.debug| is basically + not even the |minimal failing test case|. |Verbosity.debug| is basically |Verbosity.verbose| but a bit more so. You probably don't want it. Verbosity can be passed either as a |Verbosity| enum value, or as the @@ -887,17 +897,17 @@ def phases(self): Hypothesis divides tests into logically distinct phases. - - |Phase.explicit|: Running explicit examples from |@example|. - - |Phase.reuse|: Running examples from the database which previously failed. - - |Phase.generate|: Generating new random examples. - - |Phase.target|: Mutating examples for :ref:`targeted property-based + - |Phase.explicit|: Running |explicit examples| from |@example|. + - |Phase.reuse|: Running |test cases| from the database which previously failed. + - |Phase.generate|: Generating new random test cases. + - |Phase.target|: Mutating test cases for :ref:`targeted property-based testing `. Requires |Phase.generate|. - - |Phase.shrink|: Shrinking failing examples. + - |Phase.shrink|: Shrinking |failing test cases|. - |Phase.explain|: Attempting to explain why a failure occurred. Requires |Phase.shrink|. The phases argument accepts a collection with any subset of these. E.g. - ``settings(phases=[Phase.generate, Phase.shrink])`` will generate new examples + ``settings(phases=[Phase.generate, Phase.shrink])`` will generate new test cases and shrink them, but will not run explicit examples or reuse previous failures, while ``settings(phases=[Phase.explicit])`` will only run explicit examples from |@example|. @@ -920,10 +930,10 @@ def phases(self): there are no clearly suspicious lines of code, :pep:`we refuse the temptation to guess <20>`. - After shrinking to a minimal failing example, Hypothesis will try to find - parts of the example -- e.g. separate args to |@given| + After shrinking to a minimal failing test case, Hypothesis will try to find + parts of the test case -- e.g. separate args to |@given| -- which can vary freely without changing the result - of that minimal failing example. If the automated experiments run without + of that minimal failing test case. If the automated experiments run without finding a passing variation, we leave a comment in the final report: .. code-block:: python @@ -935,7 +945,7 @@ def phases(self): Just remember that the *lack* of an explanation sometimes just means that Hypothesis couldn't efficiently find one, not that no explanation (or - simpler failing example) exists. + simpler failing test case) exists. """ return self._phases @@ -947,7 +957,7 @@ def stateful_step_count(self): :ref:`stateful testing ` before we give up on finding a bug. Note that this setting is effectively multiplicative with max_examples, - as each example will run for a maximum of ``stateful_step_count`` steps. + as each test case will run for a maximum of ``stateful_step_count`` steps. The default stateful step count is ``50``. """ @@ -995,7 +1005,7 @@ def suppress_health_check(self): @property def deadline(self): """ - The maximum allowed duration of an individual test case, in milliseconds. + The maximum allowed duration of an individual |test case|, in milliseconds. You can pass an integer, float, or timedelta. If ``None``, the deadline is disabled entirely. @@ -1010,8 +1020,8 @@ def deadline(self): @property def print_blob(self): """ - If set to ``True``, Hypothesis will print code for failing examples that - can be used with |@reproduce_failure| to reproduce the failing example. + If set to ``True``, Hypothesis will print code for |failing test cases| that + can be used with |@reproduce_failure| to reproduce the failing test case. The default value is ``False``. If running on CI, the default is ``True`` instead. """ diff --git a/hypothesis/src/hypothesis/control.py b/hypothesis/src/hypothesis/control.py index 5d94150846..8118fdd9be 100644 --- a/hypothesis/src/hypothesis/control.py +++ b/hypothesis/src/hypothesis/control.py @@ -61,10 +61,10 @@ def assume(condition: object) -> Literal[True]: ... def assume(condition: object) -> Literal[True]: """Calling ``assume`` is like an :ref:`assert ` that marks - the example as bad, rather than failing the test. + the |test case| as bad, rather than failing the test. This allows you to specify properties that you *assume* will be - true, and let Hypothesis try to avoid similar examples in future. + true, and let Hypothesis try to avoid similar test cases in future. """ if _current_build_context.value is None: note_deprecation( @@ -273,7 +273,7 @@ def should_note() -> bool: def note(value: object) -> None: - """Report this value for the minimal failing example.""" + """Report this value for the |minimal failing test case|.""" if should_note(): if not isinstance(value, str): value = pretty(value) @@ -333,7 +333,7 @@ def target(observation: int | float, *, label: str = "") -> int | float: with which to guide our search for inputs that will cause an error, in addition to all the usual heuristics. Observations must always be finite. - Hypothesis will try to maximize the observed value over several examples; + Hypothesis will try to maximize the observed value over several |test cases|; almost any metric will work so long as it makes sense to increase it. For example, ``-abs(error)`` is a metric that increases as ``error`` approaches zero. @@ -351,17 +351,17 @@ def target(observation: int | float, *, label: str = "") -> int | float: ``target()`` with any label more than once per test case. .. note:: - The more examples you run, the better this technique works. + The more test cases you run, the better this technique works. As a rule of thumb, the targeting effect is noticeable above :obj:`max_examples=1000 `, - and immediately obvious by around ten thousand examples + and immediately obvious by around ten thousand test cases *per label* used by your test. :ref:`statistics` include the best score seen for each label, which can help avoid `the threshold problem `__ when the minimal - example shrinks right down to the threshold of failure (:issue:`2180`). + test case shrinks right down to the threshold of failure (:issue:`2180`). """ check_type((int, float), observation, "observation") if not math.isfinite(observation): diff --git a/hypothesis/src/hypothesis/core.py b/hypothesis/src/hypothesis/core.py index 5c8b279464..4c8572352a 100644 --- a/hypothesis/src/hypothesis/core.py +++ b/hypothesis/src/hypothesis/core.py @@ -197,11 +197,11 @@ def test_strings(s): any random inputs. |@example| may be placed in any order relative to |@given| and |@settings|. - Explicit inputs from |@example| are run in the |Phase.explicit| phase. - Explicit inputs do not count towards |settings.max_examples|. Note that - explicit inputs added by |@example| do not shrink. If an explicit input - fails, Hypothesis will stop and report the failure without generating any - random inputs. + |Explicit examples| from |@example| are run in the + |Phase.explicit| phase. Explicit examples do not count towards + |settings.max_examples|. Note that explicit examples added by |@example| do + not shrink. If an explicit example fails, Hypothesis will stop and report + the failure without generating any random inputs. |@example| can also be used to easily reproduce a failure. For instance, if Hypothesis reports that ``f(n=[0, math.nan])`` fails, you can add @@ -341,11 +341,11 @@ def seed(seed: Hashable) -> Callable[[TestFunc], TestFunc]: ``seed`` may be any hashable object. No exact meaning for ``seed`` is provided other than that for a fixed seed value Hypothesis will produce the same - examples (assuming that there are no other sources of nondeterminisim, such + |test cases| (assuming that there are no other sources of nondeterminisim, such as timing, hash randomization, or external state). For example, the following test function and |RuleBasedStateMachine| will - each generate the same series of examples each time they are executed: + each generate the same series of test cases each time they are executed: .. code-block:: python @@ -382,11 +382,11 @@ def accept(test): def reproduce_failure(version: str, blob: bytes) -> Callable[[TestFunc], TestFunc]: """ - Run the example corresponding to the binary ``blob`` in order to reproduce a + Run the |test case| corresponding to the binary ``blob`` in order to reproduce a failure. ``blob`` is a serialized version of the internal input representation of Hypothesis. - A test decorated with |@reproduce_failure| always runs exactly one example, + A test decorated with |@reproduce_failure| always runs exactly one test case, which is expected to cause a failure. If the provided ``blob`` does not cause a failure, Hypothesis will raise |DidNotReproduce|. @@ -527,7 +527,7 @@ def is_invalid_test(test, original_sig, given_arguments, given_kwargs): if empty: strats = "strategies" if len(empty) > 1 else "strategy" return invalid( - f"Cannot generate examples from empty {strats}: " + ", ".join(empty), + f"Cannot generate test cases from empty {strats}: " + ", ".join(empty), exc=Unsatisfiable, ) @@ -647,7 +647,7 @@ def execute_explicit_examples(state, wrapped_test, arguments, kwargs, original_s with contextlib.suppress(StopTest): empty_data.conclude_test(Status.INVALID) except BaseException as err: - # In order to support reporting of multiple failing examples, we yield + # In order to support reporting of multiple failing test cases, we yield # each of the (report text, error) pairs we find back to the top-level # runner. This also ensures that user-facing stack traces have as few # frames of Hypothesis internals as possible. @@ -682,9 +682,9 @@ def execute_explicit_examples(state, wrapped_test, arguments, kwargs, original_s break finally: if fragments_reported: - assert fragments_reported[0].startswith("Falsifying example") + assert fragments_reported[0].startswith("Failing test case") fragments_reported[0] = fragments_reported[0].replace( - "Falsifying example", "Falsifying explicit example", 1 + "Failing test case", "Failing explicit example", 1 ) empty_data.freeze() @@ -700,7 +700,7 @@ def execute_explicit_examples(state, wrapped_test, arguments, kwargs, original_s deliver_observation(tc) if fragments_reported: - verbose_report(fragments_reported[0].replace("Falsifying", "Trying", 1)) + verbose_report(fragments_reported[0].replace("Failing", "Trying", 1)) for f in fragments_reported[1:]: verbose_report(f) @@ -1065,9 +1065,9 @@ def run(data: ConjectureData) -> None: if print_example or current_verbosity() >= Verbosity.verbose: printer = RepresentationPrinter(context=context) if print_example: - printer.text("Falsifying example:") + printer.text("Failing test case:") else: - printer.text("Trying example:") + printer.text("Test case:") if self.print_given_args: printer.text(" ") @@ -1174,7 +1174,7 @@ def run(data: ConjectureData) -> None: report("Failed to reproduce exception. Expected: \n" + traceback) raise FlakyFailure( f"Hypothesis {text_repr} produces unreliable results: " - "Falsified on the first call but did not on a subsequent one", + "Failed on the first call but did not on a subsequent one", [exception], ) return result @@ -1555,12 +1555,12 @@ def run_engine(self): finally: ran_example.freeze() if observability_enabled(): - # log our observability line for the final failing example + # log our observability line for the final failing test case tc = make_testcase( run_start=self._start_timestamp, property=self.test_identifier, data=ran_example, - how_generated="minimal failing example", + how_generated="minimal failing test case", representation=self._string_repr, arguments=ran_example._observability_args, timing=self._timing_features, @@ -1572,12 +1572,12 @@ def run_engine(self): deliver_observation(tc) # Whether or not replay actually raised the exception again, we want - # to print the reproduce_failure decorator for the failing example. + # to print the reproduce_failure decorator for the failing test case. if self.settings.print_blob: fragments.append( - "\nYou can reproduce this example by temporarily adding " + "\nYou can reproduce this test case by temporarily adding " f"{reproduction_decorator(falsifying_example.choices)} " - "as a decorator on your test case" + "as a decorator on your test function" ) _raise_to_user( @@ -1633,7 +1633,7 @@ def _raise_to_user( errors_to_report, settings, target_lines, trailer="", *, unsound_backend=None ): """Helper function for attaching notes and grouping multiple errors.""" - failing_prefix = "Falsifying example: " + failing_prefix = "Failing test case: " ls = [] for error in errors_to_report: for note in error.fragments: @@ -1670,14 +1670,14 @@ def _raise_to_user( def fake_subTest(self, msg=None, **__): """Monkeypatch for `unittest.TestCase.subTest` during `@given`. - If we don't patch this out, each failing example is reported as a + If we don't patch this out, each failing test case is reported as a separate failing test by the unittest test runner, which is obviously incorrect. We therefore replace it for the duration with this version. """ warnings.warn( - "subTest per-example reporting interacts badly with Hypothesis " - "trying hundreds of examples, so we disable it for the duration of " + "subTest per-test-case reporting interacts badly with Hypothesis " + "trying hundreds of test cases, so we disable it for the duration of " "any test that uses `@given`.", HypothesisWarning, stacklevel=2, @@ -1717,7 +1717,7 @@ def fuzz_one_input( * If the bytestring was invalid, for example because it was too short or was filtered out by |assume| or |.filter|, |fuzz_one_input| returns ``None``. * If the bytestring was valid and the test passed, |fuzz_one_input| returns - a canonicalised and pruned bytestring which will replay that test case. + a canonicalised and pruned bytestring which will replay that |test case|. This is provided as an option to improve the performance of mutating fuzzers, but can safely be ignored. * If the test *failed*, i.e. raised an exception, |fuzz_one_input| will @@ -2201,7 +2201,7 @@ def wrapped_test(*arguments, **kwargs): and getattr(wrapped_test, "hypothesis_explicit_examples", ()) ) SKIP_BECAUSE_NO_EXAMPLES = unittest.SkipTest( - "Hypothesis has been told to run no examples for this test." + "Hypothesis has been told to run no test cases for this test." ) if not ( Phase.reuse in settings.phases or Phase.generate in settings.phases diff --git a/hypothesis/src/hypothesis/database.py b/hypothesis/src/hypothesis/database.py index 34eaecb768..de12de3ef9 100644 --- a/hypothesis/src/hypothesis/database.py +++ b/hypothesis/src/hypothesis/database.py @@ -820,7 +820,7 @@ class GitHubArtifactDatabase(ExampleDatabase): You can use this for sharing example databases between CI runs and developers, allowing the latter to get read-only access to the former. This is particularly useful for continuous fuzzing (i.e. with `HypoFuzz `_), - where the CI system can help find new failing examples through fuzzing, + where the CI system can help find new failing test cases through fuzzing, and developers can reproduce them locally without any manual effort. .. note:: diff --git a/hypothesis/src/hypothesis/errors.py b/hypothesis/src/hypothesis/errors.py index c7de51eef4..202f6356ee 100644 --- a/hypothesis/src/hypothesis/errors.py +++ b/hypothesis/src/hypothesis/errors.py @@ -50,7 +50,7 @@ def __init__(self, condition_string: str, extra: str = "") -> None: class Unsatisfiable(_Trimmable): - """We ran out of time or examples before we could find enough examples + """We ran out of time or test cases before we could find enough test cases which satisfy the assumptions of this hypothesis. This could be because the function is too slow. If so, try upping @@ -153,7 +153,7 @@ class _WrappedBaseException(Exception): class FlakyFailure(ExceptionGroup, Flaky): """ This function appears to fail non-deterministically: We have seen it - fail when passed this example at least once, but a subsequent invocation + fail when passed this value at least once, but a subsequent invocation did not fail, or caused a distinct error. Common causes for this problem are: @@ -348,7 +348,7 @@ class BackendCannotProceed(HypothesisException): The optional ``scope`` argument can enable smarter integration: verified: - Do not request further test cases from this backend. We *may* + Do not request further |test cases| from this backend. We *may* generate more test cases with other backends; if one fails then Hypothesis will report unsound verification in the backend too. diff --git a/hypothesis/src/hypothesis/extra/_patching.py b/hypothesis/src/hypothesis/extra/_patching.py index 006fd7f79f..def43f7735 100644 --- a/hypothesis/src/hypothesis/extra/_patching.py +++ b/hypothesis/src/hypothesis/extra/_patching.py @@ -13,8 +13,8 @@ Requires `hypothesis[codemods,ghostwriter]` installed, i.e. black and libcst. -This module is used by Hypothesis' builtin pytest plugin for failing examples -discovered during testing, and by HypoFuzz for _covering_ examples discovered +This module is used by Hypothesis' builtin pytest plugin for failing test cases +discovered during testing, and by HypoFuzz for _covering_ test cases discovered during fuzzing. """ @@ -98,7 +98,7 @@ def __init__( [m.Arg(m.SimpleString() & value_in_strip_via)], ) - # Codemod the failing examples to Call nodes usable as decorators + # Codemod the failing test cases to Call nodes usable as decorators self.fn_examples = { k: tuple( d diff --git a/hypothesis/src/hypothesis/internal/conjecture/engine.py b/hypothesis/src/hypothesis/internal/conjecture/engine.py index 6605242ba3..008181b902 100644 --- a/hypothesis/src/hypothesis/internal/conjecture/engine.py +++ b/hypothesis/src/hypothesis/internal/conjecture/engine.py @@ -84,7 +84,7 @@ # If the shrinking phase takes more than five minutes, abort it early and print # a warning. Many CI systems will kill a build after around ten minutes with # no output, and appearing to hang isn't great for interactive use either - -# showing partially-shrunk examples is better than quitting with no examples! +# showing partially-shrunk test cases is better than quitting with no test cases! # (but make it monkeypatchable, for the rare users who need to keep on shrinking) #: The maximum total time in seconds that the shrinker will try to shrink a failure @@ -171,9 +171,9 @@ class ExitReason(Enum): max_examples = "settings.max_examples={s.max_examples}" max_iterations = ( "settings.max_examples={s.max_examples}, " - "but < 1% of examples satisfied assumptions" + "but < 1% of test cases satisfied assumptions" ) - max_shrinks = f"shrunk example {MAX_SHRINKS} times" + max_shrinks = f"shrunk test case {MAX_SHRINKS} times" finished = "nothing left to do" flaky = "test was flaky" very_slow_shrinking = "shrinking was very slow" @@ -748,7 +748,7 @@ def _backend_cannot_proceed( # See https://github.com/HypothesisWorks/hypothesis/issues/2340 report( "WARNING: Hypothesis has spent more than five minutes working to shrink" - " a failing example, and stopped because it is making very slow" + " a failing test case, and stopped because it is making very slow" " progress. When you re-run your tests, shrinking will resume and may" " take this long before aborting again.\n\nPLEASE REPORT THIS if you can" " provide a reproducing example, so that we can improve shrinking" @@ -1022,7 +1022,7 @@ def reuse_existing_examples(self) -> None: examples that are no longer interesting are cleared out. """ if self.has_existing_examples(): - self.debug("Reusing examples from database") + self.debug("Reusing test cases from database") # We have to do some careful juggling here. We have two database # corpora: The primary and secondary. The primary corpus is a # small set of minimized examples each of which has at one point @@ -1162,12 +1162,12 @@ def generate_new_examples(self) -> None: if Phase.generate not in self.settings.phases: return if self.interesting_examples: - # The example database has failing examples from a previous run, + # The example database has failing test cases from a previous run, # so we'd rather report that they're still failing ASAP than take # the time to look for additional failures. return - self.debug("Generating new examples") + self.debug("Generating new test cases") assert self.should_generate_more() self._switch_to_hypothesis_provider = True @@ -1609,7 +1609,7 @@ def shrink_interesting_examples(self) -> None: if Phase.shrink not in self.settings.phases or not self.interesting_examples: return - self.debug("Shrinking interesting examples") + self.debug("Shrinking failing test cases") self.finish_shrinking_deadline = time.perf_counter() + MAX_SHRINKING_SECONDS for prev_data in sorted( diff --git a/hypothesis/src/hypothesis/internal/conjecture/providers.py b/hypothesis/src/hypothesis/internal/conjecture/providers.py index afba596231..08a701236c 100644 --- a/hypothesis/src/hypothesis/internal/conjecture/providers.py +++ b/hypothesis/src/hypothesis/internal/conjecture/providers.py @@ -582,7 +582,7 @@ def draw_bytes( def per_test_case_context_manager(self) -> AbstractContextManager: """ Returns a context manager which will be entered each time Hypothesis - starts generating and executing one test case, and exited when that test + starts generating and executing one |test case|, and exited when that test case finishes generating and executing, including if any exception is thrown. @@ -606,10 +606,10 @@ def realize(self, value: T, *, for_failure: bool = False) -> T: The returned value should be non-symbolic. If you cannot provide a value, raise |BackendCannotProceed| with a value of ``"discard_test_case"``. - If ``for_failure`` is ``True``, the value is associated with a failing example. + If ``for_failure`` is ``True``, the value is associated with a |failing test case|. In this case, the backend should spend substantially more effort when attempting to realize the value, since it is important to avoid discarding - failing examples. Backends may still raise |BackendCannotProceed| when + failing test cases. Backends may still raise |BackendCannotProceed| when ``for_failure`` is ``True``, if realization is truly impossible or if realization takes significantly longer than expected (say, 5 minutes). """ @@ -617,9 +617,9 @@ def realize(self, value: T, *, for_failure: bool = False) -> T: def replay_choices(self, choices: tuple[ChoiceT, ...]) -> None: """ - Called when Hypothesis has discovered a choice sequence which the provider + Called when Hypothesis has discovered a |choice sequence| which the provider may wish to enqueue to replay under its own instrumentation when we next - ask to generate a test case, rather than generating one from scratch. + ask to generate a |test case|, rather than generating one from scratch. This is used to e.g. warm-start :pypi:`hypothesis-crosshair` with a corpus of high-code-coverage inputs discovered by @@ -628,7 +628,7 @@ def replay_choices(self, choices: tuple[ChoiceT, ...]) -> None: return None def observe_test_case(self) -> dict[str, Any]: - """Called at the end of the test case when :ref:`observability + """Called at the end of the |test case| when :ref:`observability ` is enabled. The return value should be a non-symbolic json-encodable dictionary, @@ -639,7 +639,7 @@ def observe_test_case(self) -> dict[str, Any]: def observe_information_messages( self, *, lifetime: LifetimeT ) -> Iterable[_BackendInfoMsg]: - """Called at the end of each test case and again at end of the test function. + """Called at the end of each |test case| and again at end of the test function. Return an iterable of ``{type: info/alert/error, title: str, content: str | dict}`` dictionaries to be delivered as individual information messages. Hypothesis @@ -650,7 +650,7 @@ def observe_information_messages( def on_observation(self, observation: TestCaseObservation) -> None: # noqa: B027 """ - Called at the end of each test case which uses this provider, with the same + Called at the end of each |test case| which uses this provider, with the same ``observation["type"] == "test_case"`` observation that is passed to other callbacks added via |add_observability_callback|. This method is not called with ``observation["type"] in {"info", "alert", "error"}`` @@ -723,10 +723,10 @@ def span_start(self, label: int, /) -> None: # noqa: B027 # non-abstract noop Providers can track calls to |PrimitiveProvider.span_start| and |PrimitiveProvider.span_end| to learn something about the semantics of - the test's choice sequence. For instance, a provider could track the depth + the test's |choice sequence|. For instance, a provider could track the depth of the span tree, or the number of unique labels, which says something about the complexity of the choices being generated. Or a provider could track - the span tree across test cases in order to determine what strategies are + the span tree across |test cases| in order to determine what strategies are being used in what contexts. It is possible for Hypothesis to start and immediately stop a span, diff --git a/hypothesis/src/hypothesis/internal/conjecture/shrinker.py b/hypothesis/src/hypothesis/internal/conjecture/shrinker.py index c8884bcb4e..e6adf0eae9 100644 --- a/hypothesis/src/hypothesis/internal/conjecture/shrinker.py +++ b/hypothesis/src/hypothesis/internal/conjecture/shrinker.py @@ -505,18 +505,18 @@ def _explain(self) -> None: # Before we start running experiments, let's check for known inputs which would # make them redundant. The shrinking process means that we've already tried many - # variations on the minimal example, so this can save a lot of time. + # variations on the minimal test case, so this can save a lot of time. seen_passing_seq = self.engine.passing_choice_sequences( prefix=self.nodes[: min(self.shrink_target.arg_slices)[0]] ) - # Now that we've shrunk to a minimal failing example, it's time to try + # Now that we've shrunk to a minimal failing test case, it's time to try # varying each part that we've noted will go in the final report. Consider # slices in largest-first order for start, end in sorted( self.shrink_target.arg_slices, key=lambda x: (-(x[1] - x[0]), x) ): - # Check for any previous examples that match the prefix and suffix, + # Check for any previous test cases that match the prefix and suffix, # so we can skip if we found a passing example while shrinking. if any( startswith(seen, nodes[:start]) and endswith(seen, nodes[end:]) @@ -1716,7 +1716,7 @@ def minimize_individual_choices(self, chooser): x = data.draw(integers()) assert x < 10 - then in our shrunk example, x = 10 rather than say 97. + then in our shrunk test case, x = 10 rather than say 97. If we are unsuccessful at minimizing a choice of interest we then check if that's because it's changing the size of the test case and, diff --git a/hypothesis/src/hypothesis/internal/scrutineer.py b/hypothesis/src/hypothesis/internal/scrutineer.py index 1e77a9a477..4bf188d369 100644 --- a/hypothesis/src/hypothesis/internal/scrutineer.py +++ b/hypothesis/src/hypothesis/internal/scrutineer.py @@ -250,7 +250,7 @@ def get_explaining_locations(traces): EXPLANATION_STUB = ( "Explanation:", - " These lines were always and only run by failing examples:", + " These lines were always and only run by failing test cases:", ) diff --git a/hypothesis/src/hypothesis/statistics.py b/hypothesis/src/hypothesis/statistics.py index bdb29670c9..f73001aec6 100644 --- a/hypothesis/src/hypothesis/statistics.py +++ b/hypothesis/src/hypothesis/statistics.py @@ -31,7 +31,7 @@ def note_statistics(stats_dict: "StatisticsDict") -> None: def describe_targets(best_targets: dict[str, float]) -> list[str]: """Return a list of lines describing the results of `target`, if any.""" # These lines are included in the general statistics description below, - # but also printed immediately below failing examples to alleviate the + # but also printed immediately below failing test cases to alleviate the # "threshold problem" where shrinking can make severe bug look trivial. # See https://github.com/HypothesisWorks/hypothesis/issues/2180 if not best_targets: @@ -92,8 +92,9 @@ def describe_statistics(stats_dict: "StatisticsDict") -> str: lines.append( f" - during {phase} phase ({d['duration-seconds']:.2f} seconds):\n" f" - Typical runtimes: {runtime_ms}, of which {drawtime_ms} in data generation\n" - f" - {statuses['valid']} passing examples, {statuses['interesting']} " - f"failing examples, {statuses['invalid'] + statuses['overrun']} invalid examples" + f" - {statuses['valid']} passing test cases, {statuses['interesting']} " + f"failing test cases, {statuses['invalid'] + statuses['overrun']} " + "invalid test cases" ) # If we've found new distinct failures in this phase, report them distinct_failures = d["distinct-failures"] - prev_failures diff --git a/hypothesis/src/hypothesis/strategies/_internal/core.py b/hypothesis/src/hypothesis/strategies/_internal/core.py index 3e2aaf0f24..95a175774d 100644 --- a/hypothesis/src/hypothesis/strategies/_internal/core.py +++ b/hypothesis/src/hypothesis/strategies/_internal/core.py @@ -1006,7 +1006,7 @@ def randoms( are of a special HypothesisRandom subclass. - If ``note_method_calls`` is set to ``True``, Hypothesis will print the - randomly drawn values in any falsifying test case. This can be helpful + randomly drawn values in the |minimal failing test case|. This can be helpful for debugging the behaviour of randomized algorithms. - If ``use_true_random`` is set to ``True`` then values will be drawn from their usual distribution, otherwise they will actually be Hypothesis @@ -2454,18 +2454,18 @@ def test_values(data): n2 = data.draw(st.integers(min_value=n1)) assert n1 + 1 <= n2 - If the test fails, each draw will be printed with the falsifying example. + If the test fails, each draw will be printed with the |minimal failing test case|. e.g. the above is wrong (it has a boundary condition error), so will print: .. code-block:: pycon - Falsifying example: test_values(data=data(...)) + Failing test case: test_values(data=data(...)) Draw 1: 0 Draw 2: 0 Optionally, you can provide a label to identify values generated by each call to ``data.draw()``. These labels can be used to identify values in the - output of a falsifying example. + output of a failing test case. For instance: @@ -2481,7 +2481,7 @@ def test_draw_sequentially(data): .. code-block:: pycon - Falsifying example: test_draw_sequentially(data=data(...)) + Failing test case: test_draw_sequentially(data=data(...)) Draw 1 (First number): 0 Draw 2 (Second number): 0 diff --git a/hypothesis/src/hypothesis/vendor/pretty.py b/hypothesis/src/hypothesis/vendor/pretty.py index 79b59494ca..f0cde47dff 100644 --- a/hypothesis/src/hypothesis/vendor/pretty.py +++ b/hypothesis/src/hypothesis/vendor/pretty.py @@ -237,7 +237,7 @@ def __init__( We use the context to represent objects constructed by strategies by showing *how* they were constructed, and add annotations showing which parts of the - minimal failing example can vary without changing the test result. + minimal failing test case can vary without changing the test result. """ self.broken: bool = False self.output: TextIOBase = StringIO() if output is None else output @@ -268,7 +268,7 @@ def __init__( self.deferred_pprinters.update(_deferred_type_pprinters) # for which-parts-matter, we track a mapping from the (start_idx, end_idx) - # of slices into the minimal failing example; this is per-interesting_origin + # of slices into the minimal failing test case; this is per-interesting_origin # but we report each separately so that's someone else's problem here. # Invocations of self.repr_call() can report the slice for each argument, # which will then be used to look up the relevant comment if any. diff --git a/hypothesis/tests/common/utils.py b/hypothesis/tests/common/utils.py index 8e5f723825..291b07c836 100644 --- a/hypothesis/tests/common/utils.py +++ b/hypothesis/tests/common/utils.py @@ -174,8 +174,12 @@ def assert_output_contains_failure(output, test, **kwargs): assert f"{k}={v!r}" in output, (f"{k}={v!r}", output) -def assert_falsifying_output( - test, example_type="Falsifying", expected_exception=AssertionError, **kwargs +def assert_failing_output( + test, + prefix="Failing test case", + *, + expected_exception=AssertionError, + **kwargs, ): with capture_out() as out: if expected_exception is None: @@ -189,7 +193,7 @@ def assert_falsifying_output( msg = str(exc_info.value) + "\n" + notes output = out.getvalue() + msg - assert f"{example_type} example:" in output + assert f"{prefix}:" in output, (prefix, output) assert_output_contains_failure(output, test, **kwargs) @@ -335,7 +339,7 @@ def wait_for(condition, *, timeout=1, interval=0.01): ) -def run_test_for_falsifying_example(test_fn): +def run_test_for_failing_test_case(test_fn): with pytest.raises(AssertionError) as err: test_fn() return "\n".join(err.value.__notes__).strip() diff --git a/hypothesis/tests/conjecture/test_inquisitor.py b/hypothesis/tests/conjecture/test_inquisitor.py index 89aabadac7..93b460ff6e 100644 --- a/hypothesis/tests/conjecture/test_inquisitor.py +++ b/hypothesis/tests/conjecture/test_inquisitor.py @@ -40,7 +40,7 @@ def _new(): @fails_with_output(""" -Falsifying example: test_inquisitor_comments_basic_fail_if_either( +Failing test case: test_inquisitor_comments_basic_fail_if_either( # The test always failed when commented parts were varied together. a=False, # or any other generated value b=True, @@ -56,7 +56,7 @@ def test_inquisitor_comments_basic_fail_if_either(a, b, c, d, e): @fails_with_output(""" -Falsifying example: test_inquisitor_comments_basic_fail_if_not_all( +Failing test case: test_inquisitor_comments_basic_fail_if_not_all( # The test sometimes passed when commented parts were varied together. a='', # or any other generated value b='', # or any other generated value @@ -71,7 +71,7 @@ def test_inquisitor_comments_basic_fail_if_not_all(a, b, c): @fails_with_output(""" -Falsifying example: test_inquisitor_no_together_comment_if_single_argument( +Failing test case: test_inquisitor_no_together_comment_if_single_argument( a='', b='', # or any other generated value ) @@ -91,7 +91,7 @@ def ints_with_forced_draw(draw): @fails_with_output(""" -Falsifying example: test_inquisitor_doesnt_break_on_varying_forced_nodes( +Failing test case: test_inquisitor_doesnt_break_on_varying_forced_nodes( n1=100, n2=0, # or any other generated value ) @@ -111,7 +111,7 @@ def test_issue_3755_regression(start_date, data): @fails_with_output(""" -Falsifying example: test_inquisitor_no_misleading_comment_for_eq_args( +Failing test case: test_inquisitor_no_misleading_comment_for_eq_args( n1=0, n2=1, ) @@ -135,7 +135,7 @@ def __init__(self, x, y): @fails_with_output(""" -Falsifying example: test_inquisitor_builds_subargs( +Failing test case: test_inquisitor_builds_subargs( obj=MyClass( 0, # or any other generated value True, @@ -149,7 +149,7 @@ def test_inquisitor_builds_subargs(obj): @fails_with_output(""" -Falsifying example: test_inquisitor_builds_kwargs_subargs( +Failing test case: test_inquisitor_builds_kwargs_subargs( obj=MyClass( x=0, # or any other generated value y=True, @@ -163,7 +163,7 @@ def test_inquisitor_builds_kwargs_subargs(obj): @fails_with_output(""" -Falsifying example: test_inquisitor_tuple_subargs( +Failing test case: test_inquisitor_tuple_subargs( t=( 0, # or any other generated value True, @@ -177,7 +177,7 @@ def test_inquisitor_tuple_subargs(t): @fails_with_output(""" -Falsifying example: test_inquisitor_fixeddict_subargs( +Failing test case: test_inquisitor_fixeddict_subargs( d={ 'x': 0, # or any other generated value 'y': True, @@ -191,7 +191,7 @@ def test_inquisitor_fixeddict_subargs(d): @fails_with_output(""" -Falsifying example: test_inquisitor_tuple_multiple_varying( +Failing test case: test_inquisitor_tuple_multiple_varying( t=( 0, # or any other generated value '', # or any other generated value @@ -208,7 +208,7 @@ def test_inquisitor_tuple_multiple_varying(t): @fails_with_output(""" -Falsifying example: test_inquisitor_skip_subset_slices( +Failing test case: test_inquisitor_skip_subset_slices( obj=MyClass( (0, False), # or any other generated value y=False, @@ -225,7 +225,7 @@ def test_inquisitor_skip_subset_slices(obj): # Test for duplicate param names at different nesting levels @fails_with_output(""" -Falsifying example: test_inquisitor_duplicate_param_names( +Failing test case: test_inquisitor_duplicate_param_names( kw=0, # or any other generated value b={ 'kw': '', # or any other generated value @@ -255,7 +255,7 @@ def __init__(self, x): @fails_with_output(""" -Falsifying example: test_inquisitor_multi_level_nesting( +Failing test case: test_inquisitor_multi_level_nesting( bare=0, # or any other generated value outer=Outer( inner=Inner(x=0), # or any other generated value diff --git a/hypothesis/tests/conjecture/test_provider.py b/hypothesis/tests/conjecture/test_provider.py index e9eb5c2f45..7cfb87063f 100644 --- a/hypothesis/tests/conjecture/test_provider.py +++ b/hypothesis/tests/conjecture/test_provider.py @@ -464,7 +464,7 @@ def test_function(f): with capture_out() as out: test_function() - assert "Trying example: test_function(\n f=,\n)" in out.getvalue() + assert "Test case: test_function(\n f=,\n)" in out.getvalue() @pytest.mark.parametrize("verbosity", [Verbosity.verbose, Verbosity.debug]) diff --git a/hypothesis/tests/cover/__snapshots__/test_custom_reprs.ambr b/hypothesis/tests/cover/__snapshots__/test_custom_reprs.ambr index cb08d0cded..a0d9f8053c 100644 --- a/hypothesis/tests/cover/__snapshots__/test_custom_reprs.ambr +++ b/hypothesis/tests/cover/__snapshots__/test_custom_reprs.ambr @@ -1,28 +1,28 @@ # serializer version: 1 # name: test_invalid_call_syntax_falls_back_to_repr ''' - Falsifying example: inner( + Failing test case: inner( x='hello world', ) ''' # --- # name: test_map_to_bytes_prints_as_repr ''' - Falsifying example: inner( + Failing test case: inner( b=hashlib.sha256(b'').digest(), ) ''' # --- # name: test_map_to_str_prints_as_repr ''' - Falsifying example: inner( + Failing test case: inner( s='0', ) ''' # --- # name: test_reprs_as_created ''' - Falsifying example: inner( + Failing test case: inner( foo=Foo(x=1), bar=Bar(x=-1), baz=Foo(None), @@ -31,7 +31,7 @@ # --- # name: test_reprs_as_created_consistent_calls_despite_indentation ''' - Falsifying example: inner( + Failing test case: inner( a=some_foo('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'), b=Bar( some_foo( @@ -43,7 +43,7 @@ # --- # name: test_reprs_as_created_interactive ''' - Falsifying example: inner( + Failing test case: inner( data=data(...), ) Draw 1: Bar(10) diff --git a/hypothesis/tests/cover/test_custom_reprs.py b/hypothesis/tests/cover/test_custom_reprs.py index 4438b8227e..f6ba46d250 100644 --- a/hypothesis/tests/cover/test_custom_reprs.py +++ b/hypothesis/tests/cover/test_custom_reprs.py @@ -123,7 +123,7 @@ def inner(a, b): with pytest.raises(AssertionError) as err: inner() expected_re = r""" -Falsifying example: inner\( +Failing test case: inner\( a=<.*Foo object at 0x[0-9A-Fa-f]+>, b=<.*Foo object at 0x[0-9A-Fa-f]+>, \) diff --git a/hypothesis/tests/cover/test_deadline.py b/hypothesis/tests/cover/test_deadline.py index 4c762f5f59..e17c29c2e8 100644 --- a/hypothesis/tests/cover/test_deadline.py +++ b/hypothesis/tests/cover/test_deadline.py @@ -17,7 +17,7 @@ from hypothesis import given, settings, strategies as st from hypothesis.errors import DeadlineExceeded, FlakyFailure, InvalidArgument -from tests.common.utils import assert_falsifying_output, fails_with +from tests.common.utils import assert_failing_output, fails_with pytestmark = pytest.mark.skipif( settings.get_current_profile_name() == "threading", @@ -78,7 +78,7 @@ def slow_if_large(i): if i >= 1000: time.sleep(1) - assert_falsifying_output( + assert_failing_output( slow_if_large, expected_exception=DeadlineExceeded, i=1000, diff --git a/hypothesis/tests/cover/test_explicit_examples.py b/hypothesis/tests/cover/test_explicit_examples.py index 2069b4bb7e..f7a12ac9e4 100644 --- a/hypothesis/tests/cover/test_explicit_examples.py +++ b/hypothesis/tests/cover/test_explicit_examples.py @@ -28,7 +28,7 @@ from hypothesis.strategies import floats, integers, text from tests.common.utils import ( - assert_falsifying_output, + assert_failing_output, capture_out, fails_with, skipif_threading, @@ -152,7 +152,7 @@ def test_positive(x): ): test_positive() out = out.getvalue() - assert "Falsifying example: test_positive(1)" not in out + assert "Failing test case: test_positive(1)" not in out def test_prints_output_for_explicit_examples(): @@ -161,7 +161,7 @@ def test_prints_output_for_explicit_examples(): def test_positive(x): assert x > 0 - assert_falsifying_output(test_positive, "Falsifying explicit", x=-1) + assert_failing_output(test_positive, "Failing explicit example", x=-1) def test_prints_verbose_output_for_explicit_examples(): @@ -171,10 +171,10 @@ def test_prints_verbose_output_for_explicit_examples(): def test_always_passes(x): pass - assert_falsifying_output( + assert_failing_output( test_always_passes, expected_exception=None, - example_type="Trying explicit", + prefix="Trying explicit example", x="NOT AN INTEGER", ) @@ -186,7 +186,7 @@ def test_mutation(x): x.append(1) assert not x - assert_falsifying_output(test_mutation, "Falsifying explicit", x=[]) + assert_failing_output(test_mutation, "Failing explicit example", x=[]) def test_examples_are_tried_in_order(): diff --git a/hypothesis/tests/cover/test_falsifying_example_output.py b/hypothesis/tests/cover/test_failing_test_case_output.py similarity index 97% rename from hypothesis/tests/cover/test_falsifying_example_output.py rename to hypothesis/tests/cover/test_failing_test_case_output.py index 95929a08a3..a1f41d0cf5 100644 --- a/hypothesis/tests/cover/test_falsifying_example_output.py +++ b/hypothesis/tests/cover/test_failing_test_case_output.py @@ -13,7 +13,7 @@ from hypothesis import Phase, example, given, settings, strategies as st OUTPUT_WITH_BREAK = """ -Falsifying explicit example: test( +Failing explicit example: test( x={0!r}, y={0!r}, ) diff --git a/hypothesis/tests/cover/test_flakiness.py b/hypothesis/tests/cover/test_flakiness.py index 4ab30bc1fe..12e1edf208 100644 --- a/hypothesis/tests/cover/test_flakiness.py +++ b/hypothesis/tests/cover/test_flakiness.py @@ -67,7 +67,7 @@ def rude(x): first_call = False raise Nope - with pytest.raises(FlakyFailure, match="Falsified on the first call but") as e: + with pytest.raises(FlakyFailure, match="Failed on the first call but") as e: rude() exceptions = e.value.exceptions assert len(exceptions) == 1 @@ -117,7 +117,7 @@ def rude_fn(x): exec(rude_def, globals()) rude = given(integers())(rude_fn) # noqa: F821 # defined by exec() - with pytest.raises(FlakyFailure, match="Falsified on the first call but") as e: + with pytest.raises(FlakyFailure, match="Failed on the first call but") as e: rude() exceptions = e.value.exceptions assert list(map(type, exceptions)) == [ExceptionGroup] diff --git a/hypothesis/tests/cover/test_fuzz_one_input.py b/hypothesis/tests/cover/test_fuzz_one_input.py index ebbb89a1ed..b9b2d6cc50 100644 --- a/hypothesis/tests/cover/test_fuzz_one_input.py +++ b/hypothesis/tests/cover/test_fuzz_one_input.py @@ -33,7 +33,7 @@ def test_fuzz_one_input(buffer_type): # This is a standard `@given` test, which we can also use as a fuzz target. # Note that we specify the DB so we can make more precise assertions, - # and tighten the phases so we can be sure the failing examples come from fuzzing. + # and tighten the phases so we can be sure the failing test cases come from fuzzing. @given(st.text()) @settings(database=db, phases=[Phase.reuse, Phase.shrink]) def test(s): @@ -47,7 +47,7 @@ def test(s): assert len(seen) == 0 # If we run a lot of random bytestrings through fuzz_one_input, we'll eventually - # find a failing example. + # find a failing test case. with pytest.raises(AssertionError): for _ in range(1000): buf = randbytes(1000) @@ -57,7 +57,7 @@ def test(s): # fuzz_one_input returns False for invalid bytestrings, due to e.g. assume(False) assert len(seen) <= len(seeds) - # `db` contains exactly one failing example, which is either the most + # `db` contains exactly one failing test case, which is either the most # recent seed that we tried or the pruned-and-canonicalised form of it. (saved_examples,) = db.data.values() assert len(saved_examples) == 1 diff --git a/hypothesis/tests/cover/test_given_error_conditions.py b/hypothesis/tests/cover/test_given_error_conditions.py index cf2bc469e7..8795504646 100644 --- a/hypothesis/tests/cover/test_given_error_conditions.py +++ b/hypothesis/tests/cover/test_given_error_conditions.py @@ -39,7 +39,7 @@ def test_never_runs(x): with pytest.raises( Unsatisfiable, - match=r"Cannot generate examples from empty strategy: x=nothing\(\)", + match=r"Cannot generate test cases from empty strategy: x=nothing\(\)", ): test_never_runs() diff --git a/hypothesis/tests/cover/test_observability.py b/hypothesis/tests/cover/test_observability.py index ebc067877c..4675bd660d 100644 --- a/hypothesis/tests/cover/test_observability.py +++ b/hypothesis/tests/cover/test_observability.py @@ -273,7 +273,7 @@ def test_fails(x, y): } assert observation.coverage is None assert observation.features == {} - assert observation.how_generated == "minimal failing example" + assert observation.how_generated == "minimal failing test case" assert "AssertionError" in observation.metadata.traceback assert "test_fails" in observation.metadata.traceback assert observation.metadata.reproduction_decorator.startswith("@reproduce_failure") diff --git a/hypothesis/tests/cover/test_pretty.py b/hypothesis/tests/cover/test_pretty.py index e17f1c3edb..57dda4675a 100644 --- a/hypothesis/tests/cover/test_pretty.py +++ b/hypothesis/tests/cover/test_pretty.py @@ -722,7 +722,7 @@ def __repr__(self): @given(st.data()) def test_pprint_with_call_or_repr_as_call(data): - # mapped pprint repr only triggers for failing examples - which makes an + # mapped pprint repr only triggers for failing test cases - which makes an # end to end test given hypothesis difficult. fake our way around it. current_build_context().is_final = True diff --git a/hypothesis/tests/cover/test_regex.py b/hypothesis/tests/cover/test_regex.py index a28047ba64..c4922dc28a 100644 --- a/hypothesis/tests/cover/test_regex.py +++ b/hypothesis/tests/cover/test_regex.py @@ -516,7 +516,7 @@ def test(s): explain_line = " # or any other generated value" if explain else "" expected = f""" -Falsifying example: test( +Failing test case: test( s='00',{explain_line} ) """ diff --git a/hypothesis/tests/cover/test_reporting.py b/hypothesis/tests/cover/test_reporting.py index 5a3eeaa3f3..0042504aa4 100644 --- a/hypothesis/tests/cover/test_reporting.py +++ b/hypothesis/tests/cover/test_reporting.py @@ -28,7 +28,7 @@ def test_int(x): with pytest.raises(AssertionError) as err: test_int() - assert "Falsifying example" in "\n".join(err.value.__notes__) + assert "Failing test case" in "\n".join(err.value.__notes__) def test_does_not_print_debug_in_verbose(): diff --git a/hypothesis/tests/cover/test_slippage.py b/hypothesis/tests/cover/test_slippage.py index 0f4d1cad07..cf92800d6e 100644 --- a/hypothesis/tests/cover/test_slippage.py +++ b/hypothesis/tests/cover/test_slippage.py @@ -348,12 +348,12 @@ def test(x): seen.add(x) assert x - # On the first run, we look for up to ten examples: + # On the first run, we look for up to ten test cases: with pytest.raises(AssertionError): test() assert 1 < len(seen) <= MIN_TEST_CALLS - # With failing examples in the database, we stop at one. + # With failing test cases in the database, we stop at one. seen.clear() with pytest.raises(AssertionError): test() diff --git a/hypothesis/tests/cover/test_stateful.py b/hypothesis/tests/cover/test_stateful.py index 69fda3256f..1ffb2f4151 100644 --- a/hypothesis/tests/cover/test_stateful.py +++ b/hypothesis/tests/cover/test_stateful.py @@ -295,7 +295,7 @@ def fail_fast(self): run_state_machine_as_test(ProducesMultiple) # This is tightly coupled to the output format of the step printing. - # The first line is "Falsifying Example:..." the second is creating + # The first line is "Failing test case:..." the second is creating # the state machine, the third is calling the "initialize" method. assignment_line = err.value.__notes__[2] # 'populate_bundle()' returns 2 values, so should be @@ -352,7 +352,7 @@ def fail_fast(self): run_state_machine_as_test(ProducesNoVariables) # This is tightly coupled to the output format of the step printing. - # The first line is "Falsifying Example:..." the second is creating + # The first line is "Failing test case:..." the second is creating # the state machine, the third is calling the "initialize" method. assignment_line = err.value.__notes__[2] # 'populate_bundle()' returns 0 values, so there should be no @@ -732,7 +732,7 @@ def rule_1(self): result = "\n".join(err.value.__notes__) assert result == """ -Falsifying example: +Failing test case: state = BadInvariant() state.initialize_1() state.invariant_1() @@ -774,7 +774,7 @@ def rule_1(self): run_state_machine_as_test(BadRuleWithGoodInvariants) expected = """ -Falsifying example: +Failing test case: state = BadRuleWithGoodInvariants() state.invariant_1() state.initialize_1() @@ -943,7 +943,7 @@ def fail_fast(self, param): result = "\n".join(err.value.__notes__) assert result == """ -Falsifying example: +Failing test case: state = WithInitializeBundleRules() a_0 = state.initialize_a(dep='dep') state.fail_fast(param=a_0) @@ -1071,7 +1071,7 @@ def fail_eventually(self): result = "\n".join(err.value.__notes__) assert result == """ -Falsifying example: +Failing test case: state = StateMachine() state.initialize() state.fail_eventually() @@ -1091,7 +1091,7 @@ def oops(self): with pytest.raises(Failed) as err: run_state_machine_as_test(RaisesProblem) assert "\n".join(err.value.__notes__).strip() == """ -Falsifying example: +Failing test case: state = RaisesProblem() state.oops() state.teardown()""".strip() @@ -1305,7 +1305,7 @@ def fail_fast(self, param): result = "\n".join(err.value.__notes__) assert result == """ -Falsifying example: +Failing test case: state = Machine() a_0, a_1, a_2 = state.initialize() state.fail_fast(param=a_2) @@ -1354,7 +1354,7 @@ def fail_fast(self): result = "\n".join(err.value.__notes__) assert result == f""" -Falsifying example: +Failing test case: state = Machine() {repr_} state.fail_fast() @@ -1388,7 +1388,7 @@ def fail_fast(self, a1, a2, a3, b1, b2, b3): result = "\n".join(err.value.__notes__) assert result == """ -Falsifying example: +Failing test case: state = Machine() a_0, a_1, a_2 = state.initialize() b_0, b_1, b_2 = a_0, a_1, a_2 @@ -1426,7 +1426,7 @@ def fail_fast(self, a1, a2, a3, a4, a5, a6, b1, b2, b3): result = "\n".join(err.value.__notes__) assert result == """ -Falsifying example: +Failing test case: state = Machine() a_0, a_1, a_2 = state.initialize() b_0, b_1, b_2 = a_0, a_1, a_2 diff --git a/hypothesis/tests/cover/test_statistical_events.py b/hypothesis/tests/cover/test_statistical_events.py index ae07ccdeb1..73953071e3 100644 --- a/hypothesis/tests/cover/test_statistical_events.py +++ b/hypothesis/tests/cover/test_statistical_events.py @@ -213,7 +213,7 @@ def test(n): assert n < 10 stats = call_for_statistics(test) - assert "shrunk example" in stats["stopped-because"] + assert "shrunk test case" in stats["stopped-because"] def test_stateful_states_are_deduped(): @@ -264,12 +264,12 @@ def test_statistics_for_threshold_problem(): def threshold(error): target(error, label="error") assert error <= 10 - target(0.0, label="never in failing example") + target(0.0, label="never in failing test case") stats = call_for_statistics(threshold) assert " - Highest target scores:" in describe_statistics(stats) - assert "never in failing example" in describe_statistics(stats) - # Check that we report far-from-threshold failing examples + assert "never in failing test case" in describe_statistics(stats) + # Check that we report far-from-threshold failing test cases assert stats["targets"]["error"] > 10 diff --git a/hypothesis/tests/cover/test_testdecorators.py b/hypothesis/tests/cover/test_testdecorators.py index bfd09ef2b5..efe521aa63 100644 --- a/hypothesis/tests/cover/test_testdecorators.py +++ b/hypothesis/tests/cover/test_testdecorators.py @@ -44,7 +44,7 @@ from tests.common.utils import ( Why, - assert_falsifying_output, + assert_failing_output, capture_out, fails, fails_with, @@ -231,7 +231,7 @@ def test_ints_are_sorted(balthazar, evans): assume(evans >= 0) assert balthazar <= evans - assert_falsifying_output(test_ints_are_sorted, balthazar=1, evans=0) + assert_failing_output(test_ints_are_sorted, balthazar=1, evans=0) def test_does_not_print_on_success(): @@ -406,7 +406,7 @@ def test_mixed_text(x): assert set(x).issubset(set("abcdefg")) -@xfail_on_crosshair(Why.other, strict=False) # runs ~five failing examples +@xfail_on_crosshair(Why.other, strict=False) # runs ~five failing test cases def test_when_set_to_no_simplifies_runs_failing_example_twice(): failing = [] @@ -427,7 +427,7 @@ def foo(x): foo() assert len(failing) == 2 assert len(set(failing)) == 1 - assert "Falsifying example" in "\n".join(err.value.__notes__) + assert "Failing test case" in "\n".join(err.value.__notes__) assert "Lo" in err.value.__notes__ diff --git a/hypothesis/tests/cover/test_verbosity.py b/hypothesis/tests/cover/test_verbosity.py index 17ea8ee2cb..fdbda259bd 100644 --- a/hypothesis/tests/cover/test_verbosity.py +++ b/hypothesis/tests/cover/test_verbosity.py @@ -34,7 +34,7 @@ def test_works(x): pass test_works() - assert "Trying example" in o.getvalue() + assert "Test case:" in o.getvalue() def test_does_not_log_in_quiet_mode(): @@ -59,7 +59,7 @@ def test_includes_progress_in_verbose_mode(): ) out = o.getvalue() assert out - assert "Trying example: " in out + assert "Test case: " in out @xfail_on_crosshair(Why.symbolic_outside_context, strict=False) @@ -83,7 +83,7 @@ def not_first(x): foo() - assert "Trying example" in o.getvalue() + assert "Test case:" in o.getvalue() def test_includes_intermediate_results_in_verbose_mode(): @@ -102,7 +102,7 @@ def test_foo(x): test_foo() lines = o.getvalue().splitlines() - assert len([l for l in lines if "example" in l]) > 2 + assert len([l for l in lines if "Test case:" in l]) > 2 assert [l for l in lines if "AssertionError" in l] diff --git a/hypothesis/tests/nocover/test_skipping.py b/hypothesis/tests/nocover/test_skipping.py index 33fd2e68c8..b4ab2e0c94 100644 --- a/hypothesis/tests/nocover/test_skipping.py +++ b/hypothesis/tests/nocover/test_skipping.py @@ -22,10 +22,10 @@ @pytest.mark.parametrize("skip_exception", skip_exceptions_to_reraise()) -def test_no_falsifying_example_if_unittest_skip(skip_exception): +def test_no_failing_test_case_if_unittest_skip(skip_exception): """If a ``SkipTest`` exception is raised during a test, Hypothesis should not continue running the test and shrink process, nor should it print - anything about falsifying examples.""" + anything about failing test cases.""" class DemoTest(unittest.TestCase): @given(xs=integers()) @@ -39,7 +39,7 @@ def test_to_be_skipped(self, xs): suite = unittest.defaultTestLoader.loadTestsFromTestCase(DemoTest) unittest.TextTestRunner().run(suite) - assert "Falsifying example" not in o.getvalue() + assert "Failing test case" not in o.getvalue() def test_skip_exceptions_save_database_entries(): diff --git a/hypothesis/tests/nocover/test_targeting.py b/hypothesis/tests/nocover/test_targeting.py index 1165f95ee2..73cbe331df 100644 --- a/hypothesis/tests/nocover/test_targeting.py +++ b/hypothesis/tests/nocover/test_targeting.py @@ -33,7 +33,7 @@ def test_reports_target_results(testdir, multiple): script = testdir.makepyfile(TESTSUITE.format("" if multiple else "# ")) result = testdir.runpytest(script, "--tb=native", "-rN") out = "\n".join(result.stdout.lines) - assert "Falsifying example" in out + assert "Failing test case" in out assert "x=101" in out, out assert out.count("Highest target score") == 1 assert result.ret != 0 diff --git a/hypothesis/tests/patching/test_patching.py b/hypothesis/tests/patching/test_patching.py index 3dba810929..b99e71efc9 100644 --- a/hypothesis/tests/patching/test_patching.py +++ b/hypothesis/tests/patching/test_patching.py @@ -288,7 +288,7 @@ def test_pytest_reports_patch_file_location(pytester): result.assert_outcomes(failed=1) fname_pat = r"\.hypothesis/patches/\d{4}-\d\d-\d\d--[0-9a-f]{8}.patch" - pattern = f"`git apply ({fname_pat})` to add failing examples to your code\\." + pattern = f"`git apply ({fname_pat})` to add failing test cases to your code\\." print(f"{pattern=}") print(f"result.stdout=\n{indent(str(result.stdout), ' ')}") fname = re.search(pattern, str(result.stdout)).group(1) diff --git a/hypothesis/tests/pytest/test_capture.py b/hypothesis/tests/pytest/test_capture.py index ac750c81ec..28507a243c 100644 --- a/hypothesis/tests/pytest/test_capture.py +++ b/hypothesis/tests/pytest/test_capture.py @@ -33,7 +33,7 @@ def test_output_without_capture(testdir, capture, expected): result = testdir.runpytest(script, "--verbose", "--capture", capture) out = "\n".join(result.stdout.lines) assert "test_should_be_verbose" in out - assert ("Trying example" in out) == expected + assert ("Test case:" in out) == expected assert result.ret == 0 diff --git a/hypothesis/tests/pytest/test_reporting.py b/hypothesis/tests/pytest/test_reporting.py index a7cb778752..e3578fa8d0 100644 --- a/hypothesis/tests/pytest/test_reporting.py +++ b/hypothesis/tests/pytest/test_reporting.py @@ -33,7 +33,7 @@ def test_runs_reporting_hook(testdir): out = "\n".join(result.stdout.lines) assert "test_this_one_is_ok" in out assert "Captured stdout call" not in out - assert "Falsifying example" in out + assert "Failing test case" in out assert result.ret != 0 diff --git a/hypothesis/tests/pytest/test_skipping.py b/hypothesis/tests/pytest/test_skipping.py index 0baf526828..9f40d73aa4 100644 --- a/hypothesis/tests/pytest/test_skipping.py +++ b/hypothesis/tests/pytest/test_skipping.py @@ -29,22 +29,22 @@ def test_to_be_skipped(xs): """ -def test_no_falsifying_example_if_pytest_skip(testdir): +def test_no_failing_test_case_if_pytest_skip(testdir): """If ``pytest.skip() is called during a test, Hypothesis should not continue running the test and shrink process, nor should it print anything - about falsifying examples.""" + about failing test cases.""" script = testdir.makepyfile(PYTEST_TESTSUITE) result = testdir.runpytest( script, "--verbose", "--strict-markers", "-m", "hypothesis" ) out = "\n".join(result.stdout.lines) - assert "Falsifying example" not in out + assert "Failing test case" not in out def test_issue_3453_regression(testdir): """If ``pytest.skip() is called during a test, Hypothesis should not continue running the test and shrink process, nor should it print anything - about falsifying examples.""" + about failing test cases.""" script = testdir.makepyfile(""" from hypothesis import example, given, strategies as st import pytest diff --git a/hypothesis/tests/pytest/test_statistics.py b/hypothesis/tests/pytest/test_statistics.py index be8fe7cec4..a79f31c84d 100644 --- a/hypothesis/tests/pytest/test_statistics.py +++ b/hypothesis/tests/pytest/test_statistics.py @@ -53,21 +53,21 @@ def test_prints_statistics_given_option(testdir): out = get_output(testdir, TESTSUITE, PRINT_STATISTICS_OPTION) assert "Hypothesis Statistics" in out assert "max_examples=100" in out - assert "< 1% of examples satisfied assumptions" in out + assert "< 1% of test cases satisfied assumptions" in out def test_prints_statistics_given_option_under_xdist(testdir): out = get_output(testdir, TESTSUITE, PRINT_STATISTICS_OPTION, "-n", "2") assert "Hypothesis Statistics" in out assert "max_examples=100" in out - assert "< 1% of examples satisfied assumptions" in out + assert "< 1% of test cases satisfied assumptions" in out def test_prints_statistics_given_option_with_junitxml(testdir): out = get_output(testdir, TESTSUITE, PRINT_STATISTICS_OPTION, "--junit-xml=out.xml") assert "Hypothesis Statistics" in out assert "max_examples=100" in out - assert "< 1% of examples satisfied assumptions" in out + assert "< 1% of test cases satisfied assumptions" in out @skipif_threading @@ -80,7 +80,7 @@ def test_prints_statistics_given_option_under_xdist_with_junitxml(testdir): ) assert "Hypothesis Statistics" in out assert "max_examples=100" in out - assert "< 1% of examples satisfied assumptions" in out + assert "< 1% of test cases satisfied assumptions" in out UNITTEST_TESTSUITE = """ diff --git a/hypothesis/tests/snapshots/__snapshots__/test_always_failing.ambr b/hypothesis/tests/snapshots/__snapshots__/test_always_failing.ambr index 7550906368..b1271ec668 100644 --- a/hypothesis/tests/snapshots/__snapshots__/test_always_failing.ambr +++ b/hypothesis/tests/snapshots/__snapshots__/test_always_failing.ambr @@ -1,238 +1,238 @@ # serializer version: 1 # name: test_always_failing[binary] ''' - Falsifying example: inner( + Failing test case: inner( v0=b'', ) ''' # --- # name: test_always_failing[booleans] ''' - Falsifying example: inner( + Failing test case: inner( v0=False, ) ''' # --- # name: test_always_failing[builds] ''' - Falsifying example: inner( + Failing test case: inner( v0=Pair(x=0, y=''), ) ''' # --- # name: test_always_failing[builds_from_type] ''' - Falsifying example: inner( + Failing test case: inner( v0=[], ) ''' # --- # name: test_always_failing[builds_lambda_mixed_args] ''' - Falsifying example: inner( + Failing test case: inner( v0=Opaque(), ) ''' # --- # name: test_always_failing[builds_lambda_positional_args] ''' - Falsifying example: inner( + Failing test case: inner( v0=Opaque(), ) ''' # --- # name: test_always_failing[builds_lambda_returning_object] ''' - Falsifying example: inner( + Failing test case: inner( v0=Pair(0, ''), ) ''' # --- # name: test_always_failing[builds_lambda_with_defaults] ''' - Falsifying example: inner( + Failing test case: inner( v0=Opaque(), ) ''' # --- # name: test_always_failing[builds_multi_arg_lambda] ''' - Falsifying example: inner( + Failing test case: inner( v0=Opaque(), ) ''' # --- # name: test_always_failing[builds_no_arg_lambda] ''' - Falsifying example: inner( + Failing test case: inner( v0=Opaque(), ) ''' # --- # name: test_always_failing[builds_single_arg_lambda] ''' - Falsifying example: inner( + Failing test case: inner( v0=Opaque(), ) ''' # --- # name: test_always_failing[characters] ''' - Falsifying example: inner( + Failing test case: inner( v0='0', ) ''' # --- # name: test_always_failing[complex_numbers] ''' - Falsifying example: inner( + Failing test case: inner( v0=0j, ) ''' # --- # name: test_always_failing[dates] ''' - Falsifying example: inner( + Failing test case: inner( v0=datetime.date(2000, 1, 1), ) ''' # --- # name: test_always_failing[datetimes] ''' - Falsifying example: inner( + Failing test case: inner( v0=datetime.datetime(2000, 1, 1, 0, 0), ) ''' # --- # name: test_always_failing[decimals] ''' - Falsifying example: inner( + Failing test case: inner( v0=Decimal('Infinity'), ) ''' # --- # name: test_always_failing[deferred] ''' - Falsifying example: inner( + Failing test case: inner( v0=0, ) ''' # --- # name: test_always_failing[dictionaries] ''' - Falsifying example: inner( + Failing test case: inner( v0={}, ) ''' # --- # name: test_always_failing[emails] ''' - Falsifying example: inner( + Failing test case: inner( v0='0@A.AC', ) ''' # --- # name: test_always_failing[filter] ''' - Falsifying example: inner( + Failing test case: inner( v0=0, ) ''' # --- # name: test_always_failing[fixed_dictionaries] ''' - Falsifying example: inner( + Failing test case: inner( v0={'name': '', 'age': 0}, ) ''' # --- # name: test_always_failing[flatmap_lambda] ''' - Falsifying example: inner( + Failing test case: inner( v0='', ) ''' # --- # name: test_always_failing[floats] ''' - Falsifying example: inner( + Failing test case: inner( v0=0.0, ) ''' # --- # name: test_always_failing[fractions] ''' - Falsifying example: inner( + Failing test case: inner( v0=Fraction(0, 1), ) ''' # --- # name: test_always_failing[from_regex] ''' - Falsifying example: inner( + Failing test case: inner( v0='aaa', ) ''' # --- # name: test_always_failing[from_type] ''' - Falsifying example: inner( + Failing test case: inner( v0=IPv4Address('0.0.0.0'), ) ''' # --- # name: test_always_failing[frozensets] ''' - Falsifying example: inner( + Failing test case: inner( v0=frozenset(), ) ''' # --- # name: test_always_failing[functions] ''' - Falsifying example: inner( + Failing test case: inner( v0=lambda x: x, ) ''' # --- # name: test_always_failing[integers] ''' - Falsifying example: inner( + Failing test case: inner( v0=0, ) ''' # --- # name: test_always_failing[ip_addresses] ''' - Falsifying example: inner( + Failing test case: inner( v0=IPv4Address('0.0.0.0'), ) ''' # --- # name: test_always_failing[iterables] ''' - Falsifying example: inner( + Failing test case: inner( v0=iter([]), ) ''' # --- # name: test_always_failing[just] ''' - Falsifying example: inner( + Failing test case: inner( v0=42, ) ''' # --- # name: test_always_failing[lists] ''' - Falsifying example: inner( + Failing test case: inner( v0=[], ) ''' # --- # name: test_always_failing[many_args] ''' - Falsifying example: inner( + Failing test case: inner( v0=0, v1=0.0, v2='', @@ -243,35 +243,35 @@ # --- # name: test_always_failing[map_chained_lambdas_opaque] ''' - Falsifying example: inner( + Failing test case: inner( v0=Opaque(), ) ''' # --- # name: test_always_failing[map_lambda_opaque_result] ''' - Falsifying example: inner( + Failing test case: inner( v0=Opaque(), ) ''' # --- # name: test_always_failing[map_to_bytes] ''' - Falsifying example: inner( + Failing test case: inner( v0=hashlib.sha256(b'').digest(), ) ''' # --- # name: test_always_failing[map_to_str] ''' - Falsifying example: inner( + Failing test case: inner( v0='0', ) ''' # --- # name: test_always_failing[mixed_strategies] ''' - Falsifying example: inner( + Failing test case: inner( v0=[], v1={}, v2=10, @@ -280,98 +280,98 @@ # --- # name: test_always_failing[none] ''' - Falsifying example: inner( + Failing test case: inner( v0=None, ) ''' # --- # name: test_always_failing[one_of] ''' - Falsifying example: inner( + Failing test case: inner( v0=0, ) ''' # --- # name: test_always_failing[permutations] ''' - Falsifying example: inner( + Failing test case: inner( v0=[0, 1, 2, 3, 4], ) ''' # --- # name: test_always_failing[randoms] ''' - Falsifying example: inner( + Failing test case: inner( v0=HypothesisRandom(generated data), ) ''' # --- # name: test_always_failing[recursive] ''' - Falsifying example: inner( + Failing test case: inner( v0=0, ) ''' # --- # name: test_always_failing[sampled_from] ''' - Falsifying example: inner( + Failing test case: inner( v0='alice', ) ''' # --- # name: test_always_failing[sets] ''' - Falsifying example: inner( + Failing test case: inner( v0=set(), ) ''' # --- # name: test_always_failing[shared] ''' - Falsifying example: inner( + Failing test case: inner( v0=0, ) ''' # --- # name: test_always_failing[slices] ''' - Falsifying example: inner( + Failing test case: inner( v0=slice(None, None, None), ) ''' # --- # name: test_always_failing[text] ''' - Falsifying example: inner( + Failing test case: inner( v0='', ) ''' # --- # name: test_always_failing[timedeltas] ''' - Falsifying example: inner( + Failing test case: inner( v0=datetime.timedelta(0), ) ''' # --- # name: test_always_failing[times] ''' - Falsifying example: inner( + Failing test case: inner( v0=datetime.time(0, 0), ) ''' # --- # name: test_always_failing[tuples] ''' - Falsifying example: inner( + Failing test case: inner( v0=(0, '', False), ) ''' # --- # name: test_always_failing[two_args] ''' - Falsifying example: inner( + Failing test case: inner( v0=0, v1='', ) @@ -379,28 +379,28 @@ # --- # name: test_always_failing[uuids] ''' - Falsifying example: inner( + Failing test case: inner( v0=UUID('e3e70682-c209-4cac-629f-6fbed82c07cd'), ) ''' # --- # name: test_always_failing_explain[explain_from_regex] ''' - Falsifying example: inner( + Failing test case: inner( v0='00', # or any other generated value ) ''' # --- # name: test_always_failing_explain[explain_map_to_str] ''' - Falsifying example: inner( + Failing test case: inner( v0='0', # or any other generated value ) ''' # --- # name: test_always_failing_explain[map_lambda_explain_forces_call_style] ''' - Falsifying example: inner( + Failing test case: inner( v0=1, # or any other generated value ) ''' diff --git a/hypothesis/tests/snapshots/__snapshots__/test_combinators.ambr b/hypothesis/tests/snapshots/__snapshots__/test_combinators.ambr index fd4fa1b457..54204eceb3 100644 --- a/hypothesis/tests/snapshots/__snapshots__/test_combinators.ambr +++ b/hypothesis/tests/snapshots/__snapshots__/test_combinators.ambr @@ -1,7 +1,7 @@ # serializer version: 1 # name: test_data_draw ''' - Falsifying example: inner( + Failing test case: inner( data=data(...), ) Draw 1: 0 @@ -10,14 +10,14 @@ # --- # name: test_sampled_from_enum_flag ''' - Falsifying example: inner( + Failing test case: inner( c=test_sampled_from_enum_flag..Color.RED, ) ''' # --- # name: test_sampled_from_module_level_enum_flag ''' - Falsifying example: inner( + Failing test case: inner( d=Direction.NORTH, ) ''' diff --git a/hypothesis/tests/snapshots/__snapshots__/test_explain.ambr b/hypothesis/tests/snapshots/__snapshots__/test_explain.ambr index 421987e61d..9a0d9b577a 100644 --- a/hypothesis/tests/snapshots/__snapshots__/test_explain.ambr +++ b/hypothesis/tests/snapshots/__snapshots__/test_explain.ambr @@ -1,7 +1,7 @@ # serializer version: 1 # name: test_explain_builds_kwargs_subargs ''' - Falsifying example: inner( + Failing test case: inner( obj=MyClass( x=0, # or any other generated value y=True, @@ -11,7 +11,7 @@ # --- # name: test_explain_builds_subargs ''' - Falsifying example: inner( + Failing test case: inner( obj=MyClass( 0, # or any other generated value True, @@ -21,7 +21,7 @@ # --- # name: test_explain_comments_basic_fail_if_either ''' - Falsifying example: inner( + Failing test case: inner( # The test always failed when commented parts were varied together. a=False, # or any other generated value b=True, @@ -33,7 +33,7 @@ # --- # name: test_explain_comments_basic_fail_if_not_all ''' - Falsifying example: inner( + Failing test case: inner( # The test sometimes passed when commented parts were varied together. a='', # or any other generated value b='', # or any other generated value @@ -43,7 +43,7 @@ # --- # name: test_explain_duplicate_param_names ''' - Falsifying example: inner( + Failing test case: inner( kw=0, # or any other generated value b={ 'kw': '', # or any other generated value @@ -54,7 +54,7 @@ # --- # name: test_explain_fixeddict_subargs ''' - Falsifying example: inner( + Failing test case: inner( d={ 'x': 0, # or any other generated value 'y': True, @@ -64,7 +64,7 @@ # --- # name: test_explain_multi_level_nesting ''' - Falsifying example: inner( + Failing test case: inner( bare=0, # or any other generated value outer=Outer( inner=Inner(x=0), # or any other generated value @@ -75,7 +75,7 @@ # --- # name: test_explain_no_together_comment_if_single_argument ''' - Falsifying example: inner( + Failing test case: inner( a='', b='', # or any other generated value ) @@ -83,7 +83,7 @@ # --- # name: test_explain_skip_subset_slices ''' - Falsifying example: inner( + Failing test case: inner( obj=MyClass( (0, False), # or any other generated value y=False, @@ -93,7 +93,7 @@ # --- # name: test_explain_tuple_multiple_varying ''' - Falsifying example: inner( + Failing test case: inner( t=( 0, # or any other generated value '', # or any other generated value @@ -104,7 +104,7 @@ # --- # name: test_explain_tuple_subargs ''' - Falsifying example: inner( + Failing test case: inner( t=( 0, # or any other generated value True, @@ -114,7 +114,7 @@ # --- # name: test_explain_unstable_one_of_labels ''' - Falsifying example: inner( + Failing test case: inner( w=W([( 0, # or any other generated value 0, diff --git a/hypothesis/tests/snapshots/__snapshots__/test_shrinking.ambr b/hypothesis/tests/snapshots/__snapshots__/test_shrinking.ambr index f6f68cf937..ab5548881a 100644 --- a/hypothesis/tests/snapshots/__snapshots__/test_shrinking.ambr +++ b/hypothesis/tests/snapshots/__snapshots__/test_shrinking.ambr @@ -1,21 +1,21 @@ # serializer version: 1 # name: test_shrunk_float ''' - Falsifying example: inner( + Failing test case: inner( x=1.0, ) ''' # --- # name: test_shrunk_list ''' - Falsifying example: inner( + Failing test case: inner( xs=[1001], ) ''' # --- # name: test_shrunk_string ''' - Falsifying example: inner( + Failing test case: inner( s='A', ) ''' diff --git a/hypothesis/tests/snapshots/test_always_failing.py b/hypothesis/tests/snapshots/test_always_failing.py index b9ea4f4ce9..a71a7bc477 100644 --- a/hypothesis/tests/snapshots/test_always_failing.py +++ b/hypothesis/tests/snapshots/test_always_failing.py @@ -16,7 +16,7 @@ from hypothesis import given, strategies as st -from tests.common.utils import run_test_for_falsifying_example +from tests.common.utils import run_test_for_failing_test_case from tests.snapshots.conftest import EXPLAIN_SETTINGS, SNAPSHOT_SETTINGS @@ -161,7 +161,7 @@ def test_always_failing(given_args, snapshot): def inner(**kwargs): raise AssertionError - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot @pytest.mark.parametrize( @@ -183,4 +183,4 @@ def test_always_failing_explain(given_args, snapshot): def inner(**kwargs): raise AssertionError - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot diff --git a/hypothesis/tests/snapshots/test_combinators.py b/hypothesis/tests/snapshots/test_combinators.py index f8603b4d91..2daeca33e1 100644 --- a/hypothesis/tests/snapshots/test_combinators.py +++ b/hypothesis/tests/snapshots/test_combinators.py @@ -12,7 +12,7 @@ from hypothesis import given, strategies as st -from tests.common.utils import run_test_for_falsifying_example +from tests.common.utils import run_test_for_failing_test_case from tests.snapshots.conftest import SNAPSHOT_SETTINGS @@ -31,7 +31,7 @@ def inner(data): data.draw(st.text(max_size=3)) raise AssertionError - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot def test_sampled_from_enum_flag(snapshot): @@ -45,7 +45,7 @@ class Color(Flag): def inner(c): assert not (c & Color.RED) - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot def test_sampled_from_module_level_enum_flag(snapshot): @@ -54,4 +54,4 @@ def test_sampled_from_module_level_enum_flag(snapshot): def inner(d): assert not (d & Direction.NORTH) - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot diff --git a/hypothesis/tests/snapshots/test_explain.py b/hypothesis/tests/snapshots/test_explain.py index 8c31a47873..cef8e1b494 100644 --- a/hypothesis/tests/snapshots/test_explain.py +++ b/hypothesis/tests/snapshots/test_explain.py @@ -10,7 +10,7 @@ from hypothesis import given, strategies as st -from tests.common.utils import run_test_for_falsifying_example +from tests.common.utils import run_test_for_failing_test_case from tests.snapshots.conftest import EXPLAIN_SETTINGS @@ -26,7 +26,7 @@ def test_explain_comments_basic_fail_if_either(snapshot): def inner(a, b, c, d, e): assert not (b and d) - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot def test_explain_comments_basic_fail_if_not_all(snapshot): @@ -36,7 +36,7 @@ def inner(a, b, c): condition = a and b and c assert condition - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot def test_explain_no_together_comment_if_single_argument(snapshot): @@ -45,7 +45,7 @@ def test_explain_no_together_comment_if_single_argument(snapshot): def inner(a, b): assert a - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot class MyClass: @@ -60,7 +60,7 @@ def test_explain_builds_subargs(snapshot): def inner(obj): assert not obj.y - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot def test_explain_builds_kwargs_subargs(snapshot): @@ -69,7 +69,7 @@ def test_explain_builds_kwargs_subargs(snapshot): def inner(obj): assert not obj.y - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot def test_explain_tuple_subargs(snapshot): @@ -78,7 +78,7 @@ def test_explain_tuple_subargs(snapshot): def inner(t): assert not t[1] - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot def test_explain_fixeddict_subargs(snapshot): @@ -87,7 +87,7 @@ def test_explain_fixeddict_subargs(snapshot): def inner(d): assert not d["y"] - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot def test_explain_tuple_multiple_varying(snapshot): @@ -96,7 +96,7 @@ def test_explain_tuple_multiple_varying(snapshot): def inner(t): assert not t[2] - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot def test_explain_skip_subset_slices(snapshot): @@ -105,7 +105,7 @@ def test_explain_skip_subset_slices(snapshot): def inner(obj): assert obj.y - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot def test_explain_duplicate_param_names(snapshot): @@ -117,7 +117,7 @@ def test_explain_duplicate_param_names(snapshot): def inner(kw, b): assert not b["c"] - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot class Outer: @@ -142,7 +142,7 @@ def test_explain_multi_level_nesting(snapshot): def inner(bare, outer): assert not outer.value - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot # Regression for https://github.com/HypothesisWorks/hypothesis/issues/4708. @@ -168,4 +168,4 @@ def test_explain_unstable_one_of_labels(snapshot): def inner(w): assert not isinstance(w.a, list) - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot diff --git a/hypothesis/tests/snapshots/test_shrinking.py b/hypothesis/tests/snapshots/test_shrinking.py index 837bb1afd8..5df8e7165c 100644 --- a/hypothesis/tests/snapshots/test_shrinking.py +++ b/hypothesis/tests/snapshots/test_shrinking.py @@ -10,7 +10,7 @@ from hypothesis import given, strategies as st -from tests.common.utils import run_test_for_falsifying_example +from tests.common.utils import run_test_for_failing_test_case from tests.snapshots.conftest import SNAPSHOT_SETTINGS @@ -20,7 +20,7 @@ def test_shrunk_list(snapshot): def inner(xs): assert sum(xs) <= 1000 - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot def test_shrunk_string(snapshot): @@ -29,7 +29,7 @@ def test_shrunk_string(snapshot): def inner(s): assert s == s.lower() - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot def test_shrunk_float(snapshot): @@ -38,4 +38,4 @@ def test_shrunk_float(snapshot): def inner(x): assert x <= 0.5 - assert run_test_for_falsifying_example(inner) == snapshot + assert run_test_for_failing_test_case(inner) == snapshot diff --git a/tooling/src/hypothesistooling/__main__.py b/tooling/src/hypothesistooling/__main__.py index 94917a4b5f..089e825532 100644 --- a/tooling/src/hypothesistooling/__main__.py +++ b/tooling/src/hypothesistooling/__main__.py @@ -55,6 +55,7 @@ create_github_release, get_autoupdate_message, has_release, + install_hypothesis_editable, tag_name, update_changelog_and_version, upload_distribution_to_pypi, @@ -663,10 +664,17 @@ def documentation(): try: if has_release(): update_changelog_and_version() + install_hypothesis_editable() build_docs() finally: subprocess.check_call( - ["git", "checkout", "docs/changelog.rst", "src/hypothesis/version.py"], + [ + "git", + "checkout", + "docs/changelog.rst", + "rust/Cargo.toml", + "rust/Cargo.lock", + ], cwd=HYPOTHESIS, ) @@ -686,6 +694,7 @@ def live_website(): @task() def live_docs(): + install_hypothesis_editable() pip_tool( "sphinx-autobuild", "docs", @@ -976,15 +985,7 @@ def check_whole_repo_tests(*args): @task() def check_documentation(*args): install.ensure_shellcheck() - install.ensure_rustc(ci_version_rust) - # Here is why -e is necessary: our docs build prepends src/ onto sys.path so the local - # source code is consulted first. Without -e, any rust code is compiled into site-packages, - # which the src/ prepending will not reference. -e causes rust code to be compiled - # into src/, which lets our sys.path edit pick it up. - subprocess.check_call( - [sys.executable, "-m", "pip", "install", "--upgrade", "-e", HYPOTHESIS], - env={**os.environ, **RUST_BUILD_ENV}, - ) + install_hypothesis_editable() if not args: args = ["-n", "auto", REPO_TESTS / "documentation"] diff --git a/tooling/src/hypothesistooling/release.py b/tooling/src/hypothesistooling/release.py index 3376ec1a2c..07b56021bb 100644 --- a/tooling/src/hypothesistooling/release.py +++ b/tooling/src/hypothesistooling/release.py @@ -124,6 +124,18 @@ def has_source_changes(): return has_changes([PYTHON_SRC]) +def install_hypothesis_editable(): + # Here is why -e is necessary: our docs build prepends src/ onto sys.path so the local + # source code is consulted first. Without -e, any rust code is compiled into site-packages, + # which the src/ prepending will not reference. -e causes rust code to be compiled + # into src/, which lets our sys.path edit pick it up. + install.ensure_rustc(ci_version_rust) + subprocess.check_call( + [sys.executable, "-m", "pip", "install", "--upgrade", "-e", HYPOTHESIS], + env={**os.environ, **RUST_BUILD_ENV}, + ) + + def build_docs(*, builder="html", only=(), to=None): # See https://www.sphinx-doc.org/en/stable/man/sphinx-build.html pip_tool( @@ -287,13 +299,7 @@ def upload_distribution_to_pypi(*, expected_version): def create_github_release(): - # Building the docs requires hypothesis installed editably. See check_documentation - # comment for details of why. - install.ensure_rustc(ci_version_rust) - subprocess.check_call( - [sys.executable, "-m", "pip", "install", "--upgrade", "-e", HYPOTHESIS], - env={**os.environ, **RUST_BUILD_ENV}, - ) + install_hypothesis_editable() # Construct plain-text + markdown version of this changelog entry, # with link to canonical source. build_docs(builder="text", only=["docs/changelog.rst"]) diff --git a/website/content/2016-04-16-anatomy-of-a-test.md b/website/content/2016-04-16-anatomy-of-a-test.md index 6d81147bae..a0093732af 100644 --- a/website/content/2016-04-16-anatomy-of-a-test.md +++ b/website/content/2016-04-16-anatomy-of-a-test.md @@ -41,7 +41,7 @@ E assert (0.0 + nan) == (nan + 0.0) test_floats.py:7: AssertionError -Falsifying example: test_floats_are_commutative(x=0.0, y=nan) +Failing test case: test_floats_are_commutative(x=0.0, y=nan) ``` The test fails, because [nan](https://en.wikipedia.org/wiki/NaN) is a valid floating @@ -78,40 +78,40 @@ Whichever one we choose, running it we'll see output something like the followin ``` -Trying example: test_floats_are_commutative(x=-0.05851890381391768, y=-6.060045836901702e+300) -Trying example: test_floats_are_commutative(x=-0.06323690311413645, y=2.0324087421708266e-308) -Trying example: test_floats_are_commutative(x=-0.05738038380011458, y=-1.5993500302384265e-308) -Trying example: test_floats_are_commutative(x=-0.06598754758697359, y=-1.1412902232349034e-308) -Trying example: test_floats_are_commutative(x=-0.06472919559855002, y=1.7429441378277974e+35) -Trying example: test_floats_are_commutative(x=-0.06537123121982172, y=-8.136220566134233e-156) -Trying example: test_floats_are_commutative(x=-0.06016703321602157, y=1.9718842567475311e-215) -Trying example: test_floats_are_commutative(x=-0.055257588875432875, y=1.578407827448836e-308) -Trying example: test_floats_are_commutative(x=-0.06313031758042666, y=1.6749023021600297e-175) -Trying example: test_floats_are_commutative(x=-0.05886897920547916, y=1.213699633272585e+292) -Trying example: test_floats_are_commutative(x=-12.0, y=-0.0) -Trying example: test_floats_are_commutative(x=4.0, y=1.7976931348623157e+308) -Trying example: test_floats_are_commutative(x=-9.0, y=0.0) -Trying example: test_floats_are_commutative(x=-38.0, y=1.7976931348623157e+308) -Trying example: test_floats_are_commutative(x=-24.0, y=1.5686642754811104e+289) -Trying example: test_floats_are_commutative(x=-10.0, y=nan) +Test case: test_floats_are_commutative(x=-0.05851890381391768, y=-6.060045836901702e+300) +Test case: test_floats_are_commutative(x=-0.06323690311413645, y=2.0324087421708266e-308) +Test case: test_floats_are_commutative(x=-0.05738038380011458, y=-1.5993500302384265e-308) +Test case: test_floats_are_commutative(x=-0.06598754758697359, y=-1.1412902232349034e-308) +Test case: test_floats_are_commutative(x=-0.06472919559855002, y=1.7429441378277974e+35) +Test case: test_floats_are_commutative(x=-0.06537123121982172, y=-8.136220566134233e-156) +Test case: test_floats_are_commutative(x=-0.06016703321602157, y=1.9718842567475311e-215) +Test case: test_floats_are_commutative(x=-0.055257588875432875, y=1.578407827448836e-308) +Test case: test_floats_are_commutative(x=-0.06313031758042666, y=1.6749023021600297e-175) +Test case: test_floats_are_commutative(x=-0.05886897920547916, y=1.213699633272585e+292) +Test case: test_floats_are_commutative(x=-12.0, y=-0.0) +Test case: test_floats_are_commutative(x=4.0, y=1.7976931348623157e+308) +Test case: test_floats_are_commutative(x=-9.0, y=0.0) +Test case: test_floats_are_commutative(x=-38.0, y=1.7976931348623157e+308) +Test case: test_floats_are_commutative(x=-24.0, y=1.5686642754811104e+289) +Test case: test_floats_are_commutative(x=-10.0, y=nan) Traceback (most recent call last): ... AssertionError: assert (-10.0 + nan) == (nan + -10.0) -Trying example: test_floats_are_commutative(x=10.0, y=nan) +Test case: test_floats_are_commutative(x=10.0, y=nan) Traceback (most recent call last): ... AssertionError: assert (10.0 + nan) == (nan + 10.0) -Trying example: test_floats_are_commutative(x=0.0, y=nan) +Test case: test_floats_are_commutative(x=0.0, y=nan) Traceback (most recent call last): ... AssertionError: assert (0.0 + nan) == (nan + 0.0) -Trying example: test_floats_are_commutative(x=0.0, y=0.0) -Trying example: test_floats_are_commutative(x=0.0, y=inf) -Trying example: test_floats_are_commutative(x=0.0, y=-inf) -Falsifying example: test_floats_are_commutative(x=0.0, y=nan) +Test case: test_floats_are_commutative(x=0.0, y=0.0) +Test case: test_floats_are_commutative(x=0.0, y=inf) +Test case: test_floats_are_commutative(x=0.0, y=-inf) +Failing test case: test_floats_are_commutative(x=0.0, y=nan) ``` Notice how the first failing example we got was -10.0, nan but Hypothesis was able @@ -121,30 +121,30 @@ Hypothesis's output easy to understand. ``` -Trying example: test_floats_are_commutative(x=nan, y=0.0) +Test case: test_floats_are_commutative(x=nan, y=0.0) Traceback (most recent call last): ... AssertionError: assert (nan + 0.0) == (0.0 + nan) -Trying example: test_floats_are_commutative(x=0.0, y=0.0) -Trying example: test_floats_are_commutative(x=inf, y=0.0) -Trying example: test_floats_are_commutative(x=-inf, y=0.0) -Falsifying example: test_floats_are_commutative(x=nan, y=0.0) +Test case: test_floats_are_commutative(x=0.0, y=0.0) +Test case: test_floats_are_commutative(x=inf, y=0.0) +Test case: test_floats_are_commutative(x=-inf, y=0.0) +Failing test case: test_floats_are_commutative(x=nan, y=0.0) ``` Now lets see what happens when we rerun the test: ``` -Trying example: test_floats_are_commutative(x=0.0, y=nan) +Test case: test_floats_are_commutative(x=0.0, y=nan) Traceback (most recent call last): ... AssertionError: assert (0.0 + nan) == (nan + 0.0) -Trying example: test_floats_are_commutative(x=0.0, y=0.0) -Trying example: test_floats_are_commutative(x=0.0, y=inf) -Trying example: test_floats_are_commutative(x=0.0, y=-inf) -Falsifying example: test_floats_are_commutative(x=0.0, y=nan) +Test case: test_floats_are_commutative(x=0.0, y=0.0) +Test case: test_floats_are_commutative(x=0.0, y=inf) +Test case: test_floats_are_commutative(x=0.0, y=-inf) +Failing test case: test_floats_are_commutative(x=0.0, y=nan) ``` Notice how the first example it tried was the failing example we had last time? That's @@ -171,11 +171,11 @@ def test_floats_are_commutative(x, y): ``` ``` -Falsifying example: test_floats_are_commutative(x=0.0, y=nan) +Failing test case: test_floats_are_commutative(x=0.0, y=nan) ``` If you run this in verbose mode it will print out -Falsifying example: test\_floats\_are\_commutative(x=0.0, y=nan) immediately and +Failing test case: test\_floats\_are\_commutative(x=0.0, y=nan) immediately and not try to do any shrinks. Values you pass in via example will not be shrunk. This is partly a technical limitation but it can often be useful as well. diff --git a/website/content/2016-04-16-encode-decode-invariant.md b/website/content/2016-04-16-encode-decode-invariant.md index 0a1cec1fe9..400b878cd1 100644 --- a/website/content/2016-04-16-encode-decode-invariant.md +++ b/website/content/2016-04-16-encode-decode-invariant.md @@ -70,7 +70,7 @@ fuzzing: The code does not correctly handle the empty string. ``` -Falsifying example: test_decode_inverts_encode(s='') +Failing test case: test_decode_inverts_encode(s='') UnboundLocalError: local variable 'character' referenced before assignment ``` @@ -125,7 +125,7 @@ E + 110 test_encoding.py:35: AssertionError ------------------------------------ Hypothesis ------------------------------------ -Falsifying example: test_decode_inverts_encode(s='110') +Failing test case: test_decode_inverts_encode(s='110') ``` diff --git a/website/content/2016-04-19-rule-based-stateful-testing.md b/website/content/2016-04-19-rule-based-stateful-testing.md index 5f45bb0680..0227a51e36 100644 --- a/website/content/2016-04-19-rule-based-stateful-testing.md +++ b/website/content/2016-04-19-rule-based-stateful-testing.md @@ -86,7 +86,7 @@ E Use -v to get the full diff binheap.py:74: AssertionError ----- Hypothesis ----- -Falsifying example: test_pop_in_sorted_order(ls=[0, 1, 0]) +Failing test case: test_pop_in_sorted_order(ls=[0, 1, 0]) ``` So we replace heappop with a correct implementation which rebalances the heap: diff --git a/website/content/2016-04-29-testing-performance-optimizations.md b/website/content/2016-04-29-testing-performance-optimizations.md index 88192ad5dd..ab0aae2c41 100644 --- a/website/content/2016-04-29-testing-performance-optimizations.md +++ b/website/content/2016-04-29-testing-performance-optimizations.md @@ -95,7 +95,7 @@ E Use -v to get the full diff foo.py:43: AssertionError ----- Hypothesis ----- -Falsifying example: test_bubble_sorting_is_same_as_merge_sorting(ls=[0, 0]) +Failing test case: test_bubble_sorting_is_same_as_merge_sorting(ls=[0, 0]) ``` What's happened is that we messed up our implementation of merge\_sorted\_lists, because we forgot diff --git a/website/content/2016-05-29-testing-optimizers-with-hypothesis.md b/website/content/2016-05-29-testing-optimizers-with-hypothesis.md index 85a28aea6c..1958c36684 100644 --- a/website/content/2016-05-29-testing-optimizers-with-hypothesis.md +++ b/website/content/2016-05-29-testing-optimizers-with-hypothesis.md @@ -123,7 +123,7 @@ In fact, both of these tests fail: ``` -Falsifying example: test_cloning_an_item(items=[(1, 1), (1, 1), (2, 5)], capacity=7, data=data(...)) +Failing test case: test_cloning_an_item(items=[(1, 1), (1, 1), (2, 5)], capacity=7, data=data(...)) Draw 1: (1, 1) ``` @@ -135,7 +135,7 @@ items that are small enough to fit in it. ``` -Falsifying example: test_removing_a_chosen_item(items=[(1, 1), (2, 4), (1, 2)], capacity=6, data=data(...)) +Failing test case: test_removing_a_chosen_item(items=[(1, 1), (2, 4), (1, 2)], capacity=6, data=data(...)) Draw 1: (1, 1) ``` diff --git a/website/content/2016-06-13-testing-configuration-parameters.md b/website/content/2016-06-13-testing-configuration-parameters.md index 5d6d478551..f76bdc033d 100644 --- a/website/content/2016-06-13-testing-configuration-parameters.md +++ b/website/content/2016-06-13-testing-configuration-parameters.md @@ -126,14 +126,14 @@ In both cases, a password would no longer validate against itself: ``` -Falsifying example: test_a_password_verifies( +Failing test case: test_a_password_verifies( password='', time_cost=1, parallelism=1, memory_cost=8, hash_len=4, salt_len=8, ) ``` ``` -Falsifying example: test_a_password_verifies( +Failing test case: test_a_password_verifies( password='', time_cost=1, parallelism=1, memory_cost=8, hash_len=513, salt_len=8 ) diff --git a/website/content/2016-06-30-tests-as-complete-specifications.md b/website/content/2016-06-30-tests-as-complete-specifications.md index c34834a6ea..e972d275ca 100644 --- a/website/content/2016-06-30-tests-as-complete-specifications.md +++ b/website/content/2016-06-30-tests-as-complete-specifications.md @@ -103,7 +103,7 @@ And sure enough, if you run the test enough times it eventually *does* fail: ``` -Falsifying example: test_inserting_at_smaller_index_gives_unsorted( +Failing test case: test_inserting_at_smaller_index_gives_unsorted( ls=[0, 1, 1, 1, 1], v=1 ) ``` diff --git a/website/content/2016-07-04-calculating-the-mean.md b/website/content/2016-07-04-calculating-the-mean.md index 7e45f0cad6..76e08c4abd 100644 --- a/website/content/2016-07-04-calculating-the-mean.md +++ b/website/content/2016-07-04-calculating-the-mean.md @@ -51,7 +51,7 @@ assert inf <= 8.98846567431158e+307 + where inf = mean([8.988465674311579e+307, 8.98846567431158e+307]) + and 8.98846567431158e+307 = max([8.988465674311579e+307, 8.98846567431158e+307]) -Falsifying example: test_mean_is_within_reasonable_bounds( +Failing test case: test_mean_is_within_reasonable_bounds( ls=[8.988465674311579e+307, 8.98846567431158e+307] ) ``` @@ -74,7 +74,7 @@ assert 1.390671161567e-309 <= 1.390671161566996e-309 where 1.390671161567e-309 = min([1.390671161567e-309, 1.390671161567e-309, 1.390671161567e-309]) and 1.390671161566996e-309 = mean([1.390671161567e-309, 1.390671161567e-309, 1.390671161567e-309]) -Falsifying example: test_mean_is_within_reasonable_bounds( +Failing test case: test_mean_is_within_reasonable_bounds( ls=[1.390671161567e-309, 1.390671161567e-309, 1.390671161567e-309] ) ``` @@ -107,7 +107,7 @@ assert inf <= 8.98846567431158e+307 where inf = mean([8.988465674311579e+307, 8.98846567431158e+307]) and 8.98846567431158e+307 = max([8.988465674311579e+307, 8.98846567431158e+307]) -Falsifying example: test_mean_is_within_reasonable_bounds( +Failing test case: test_mean_is_within_reasonable_bounds( ls=[8.988465674311579e+307, 8.98846567431158e+307] ) ``` @@ -119,7 +119,7 @@ this is broken too ``` OverflowError: integer division result too large for a float -Falsifying example: test_mean_is_within_reasonable_bounds( +Failing test case: test_mean_is_within_reasonable_bounds( ls=[8.988465674311579e+307, 8.98846567431158e+307] ) ``` diff --git a/website/content/2016-07-23-what-is-hypothesis.md b/website/content/2016-07-23-what-is-hypothesis.md index 1b4377f752..644e541f00 100644 --- a/website/content/2016-07-23-what-is-hypothesis.md +++ b/website/content/2016-07-23-what-is-hypothesis.md @@ -90,7 +90,7 @@ When this is first run, you will see an error that looks something like this: ``` -Falsifying example: test_decode_inverts_encode(s='\xc2\xc2\x80') +Failing test case: test_decode_inverts_encode(s='\xc2\xc2\x80') Traceback (most recent call last): File "/home/david/.pyenv/versions/2.7/lib/python2.7/site-packages/hypothesis/core.py", line 443, in evaluate_test_data diff --git a/website/content/2017-03-09-hypothesis-for-researchers.md b/website/content/2017-03-09-hypothesis-for-researchers.md index c87c80715b..bc1739fc60 100644 --- a/website/content/2017-03-09-hypothesis-for-researchers.md +++ b/website/content/2017-03-09-hypothesis-for-researchers.md @@ -99,7 +99,7 @@ Then on running we would see the following output: ---- Hypothesis ---- - Falsifying example: test_sort_is_idempotent(ls=[0, 1]) + Failing test case: test_sort_is_idempotent(ls=[0, 1]) ``` Hypothesis probably started with a much more complicated example (the test fails for essentially any list with more @@ -135,7 +135,7 @@ This fails because we've forgotten than `i` may be zero, and also about Python's ---- Hypothesis ---- - Falsifying example: test_sort_is_idempotent(ls=[0, 1], data=data(...)) + Failing test case: test_sort_is_idempotent(ls=[0, 1], data=data(...)) Draw 1: 0 ``` diff --git a/website/content/2017-09-14-multi-bug-discovery.md b/website/content/2017-09-14-multi-bug-discovery.md index 61a624ac4c..637f9d5362 100644 --- a/website/content/2017-09-14-multi-bug-discovery.md +++ b/website/content/2017-09-14-multi-bug-discovery.md @@ -63,7 +63,7 @@ Well, as of Hypothesis 3.29.0, released a few weeks ago, now it does! If you run the above test now, you'll get the following: ``` -Falsifying example: test(ls=[nan]) +Failing test case: test(ls=[nan]) Traceback (most recent call last): File "/home/david/hypothesis-python/src/hypothesis/core.py", line 671, in run print_example=True, is_final=True @@ -79,7 +79,7 @@ Traceback (most recent call last): assert min(ls) <= mean(ls) <= max(ls) AssertionError -Falsifying example: test(ls=[]) +Failing test case: test(ls=[]) Traceback (most recent call last): File "/home/david/hypothesis-python/src/hypothesis/core.py", line 671, in run print_example=True, is_final=True diff --git a/website/content/2018-01-08-smarkets.md b/website/content/2018-01-08-smarkets.md index 1215455603..ac2e683b03 100644 --- a/website/content/2018-01-08-smarkets.md +++ b/website/content/2018-01-08-smarkets.md @@ -121,7 +121,7 @@ Over all I'm much happier with the new health check system and think it does a m Historically output from Hypothesis has looked something like this: ``` -Falsifying example: test_is_minimal(ls=[0], v=1) +Failing test case: test_is_minimal(ls=[0], v=1) ``` Or, if you had a failed health check, like the following: