How to control test execution result in Junit5?

Question

I created a new annotation in JUnit5, that creates a stream of replication of the same test, and runs them or disables them according to some conditions.

But, if at least one of the iterations fails it automatically fails the whole test suite, and I want to be able to control the parent test execution result.

For example, I want to set that if a certain number of replicas have passed then the whole suite should pass. Is there any way to do this?

Here is my code:

public class Test {

private static int i = 0;

    @FlakyTest(maxIterations = 10, maxFailuresRate = 0.4)
    public void test() {
        if(i++ == 0){
            assert false;
        } else {
            assert true;
        }
    }
}


import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.api.extension.ExtendWith;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

@Target({ElementType.METHOD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
@TestTemplate
@ExtendWith(FlakyTestRunner.class)
public @interface FlakyTest {

    int maxIterations() default 6;
    double maxFailuresRate() default 0.2;
}


import org.junit.jupiter.api.extension.AfterTestExecutionCallback;
import org.junit.jupiter.api.extension.ConditionEvaluationResult;
import org.junit.jupiter.api.extension.ExecutionCondition;
import org.junit.jupiter.api.extension.Extension;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.TestTemplateInvocationContext;
import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import static flaky.FlakyTestRunner.didPassedFailureRate;
import static org.junit.platform.commons.util.AnnotationUtils.isAnnotated;

public class FlakyTestRunner implements TestTemplateInvocationContextProvider, AfterTestExecutionCallback {

    public static int iteration = 0;
    public static int maxIterations;
    public static double maxFailuresRate;
    private static Map iterationsResultsMap = new HashMap<>();

    @Override
    public boolean supportsTestTemplate(ExtensionContext extensionContext) {
        return isAnnotated(extensionContext.getTestMethod(), FlakyTest.class);
    }

    @Override
    public Stream provideTestTemplateInvocationContexts(ExtensionContext extensionContext) {
        maxIterations = extensionContext.getElement().get().getAnnotation(FlakyTest.class).maxIterations();
        maxFailuresRate = extensionContext.getElement().get().getAnnotation(FlakyTest.class).maxFailuresRate();
        List invocationContexts = new ArrayList();
        for (int i = 0; i < maxIterations; i++) {
            invocationContexts.add(new FlakyIterationRunnerTemplateInvocationContext());
        }
        return invocationContexts.stream();
    }

    @Override
    public void afterTestExecution(ExtensionContext extensionContext) {
        iterationsResultsMap.put(iteration, !extensionContext.getExecutionException().isPresent());
    }

    public static boolean didPassedFailureRate() {
        if (iteration > 2) {
            return getFailedTestsRate() >= maxFailuresRate;
        }
        return false;
    }

    private static double getFailedTestsRate() {
        int sum = iterationsResultsMap.values()
                                      .stream()
                                      .mapToInt(successFlag -> successFlag ? 0 : 1)
                                      .sum();
        return ((double) sum) / maxIterations;
    }
}

class FlakyIterationRunnerTemplateInvocationContext implements TestTemplateInvocationContext {

    @Override
    public List getAdditionalExtensions() {
        List extensions = new ArrayList<>();
        extensions.add(new FlakyIterationRunnerExecutionCondition());
        return extensions;
    }
}

class FlakyIterationRunnerExecutionCondition implements ExecutionCondition {

    @Override
    public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext extensionContext) {
        FlakyTestRunner.iteration++;
        if (FlakyTestRunner.iteration <= FlakyTestRunner.maxIterations && !didPassedFailureRate()) {
            return ConditionEvaluationResult.enabled("Passed");
        }
        return ConditionEvaluationResult.disabled("Iteration number: " + FlakyTestRunner.iteration + ", did passed failure rate? " + didPassedFailureRate()
                + ". Max failures rate allowed - " + FlakyTestRunner.maxFailuresRate);
    }
}

user_10651515 · Answer

I generally think "flakey tests" are a bad idea and some level of refactoring can nearly always be done to remove the need for things like this. However if this is the approach you want to take I think you can implement it by having your extension implement TestExecutionExceptionHandler so that you can decide when the AssertionError should cause test failure. I don't know if this is a particularly good implementation but I guess this would do what you want it to:

public class FlakyTestRunner implements TestTemplateInvocationContextProvider, TestExecutionExceptionHandler  {

    private Map results = new HashMap<>();

    @Override
    public boolean supportsTestTemplate(ExtensionContext extensionContext) {
        return isAnnotated(extensionContext.getTestMethod(), FlakyTest.class);
    }

    @Override
    public Stream provideTestTemplateInvocationContexts(
        ExtensionContext extensionContext) {
        int maxIterations = extensionContext.getElement().get().getAnnotation(FlakyTest.class).maxIterations();
        double maxFailuresRate = extensionContext.getElement().get().getAnnotation(FlakyTest.class).maxFailuresRate();
        results.put(extensionContext.getTestMethod().get(), new MultiIterationResult(maxIterations, maxFailuresRate));
        List invocationContexts = new ArrayList();
        for (int i = 0; i < maxIterations; i++) {
            invocationContexts.add(new FlakyIterationRunnerTemplateInvocationContext());
        }
        return invocationContexts.stream();
    }

    @Override
    public void handleTestExecutionException(ExtensionContext context, Throwable throwable) throws Throwable {
        results.get(context.getTestMethod().orElseThrow(() -> throwable)).handleFailure(throwable);
    }

    private class MultiIterationResult {
        private final int iterations;
        private final double failureThreshold;
        private int failCount = 0;

        public MultiIterationResult(int iterations, double failureThreshold) {
            this.iterations = iterations;
            this.failureThreshold = failureThreshold;
        }

        public void handleFailure(Throwable throwable) throws Throwable {
            failCount++;
            if((double)failCount/iterations > failureThreshold) {
                throw throwable;
            }
        }
    }

    private  class FlakyIterationRunnerTemplateInvocationContext implements TestTemplateInvocationContext {
    }
}

How to control test execution result in Junit5?

1 Answers