I'm trying to create a pytest plugin that runs a test multiple times to see if it ever passes, skipping on failures unless it's the last run. I use a custom marker for those tests, then pytest_generate_tests to parametrize the test, then modify the result in pytest_pyfunc_call. However, with this approach, the test function needs to actually specify the parameter name, is there some way to hide this parameter from the actual function?
Here's the code for the plugin:
from typing import Optional
import pytest
from _pytest.python import Metafunc, Function
from pluggy.callers import _Result
def pytest_configure(config):
config.addinivalue_line(
"markers", "brute_force: Brute force the test multiple times."
)
class Memory:
def __init__(self, attempts: int):
self.attempts = attempts
self.attempt = 0
self.has_passed = False
def pytest_generate_tests(metafunc: Metafunc):
marker = metafunc.definition.get_closest_marker("brute_force")
if not marker:
return
num_attempts = marker.args and marker.args[0] or 3
memory = Memory(num_attempts)
metafunc.parametrize("memory", [memory for _ in range(num_attempts)])
@pytest.hookimpl(hookwrapper=True)
def pytest_pyfunc_call(pyfuncitem: Function):
memory: Optional[Memory] = pyfuncitem.callspec.params.pop('memory', None)
if memory is not None:
memory.attempt +=1
if memory.has_passed:
raise pytest.skip("already passed")
outcome: _Result = yield
if memory is not None:
if outcome.excinfo is None:
memory.has_passed = True
elif memory.attempt != memory.attempts:
pytest.skip("may pass later")
And here is an example of a test, I want to not have the function to have to specify memory as an argument:
import pytest
num = -1
@pytest.mark.brute_force(3)
def test_simple(memory):
global num
num += 1
assert 10 / num