Skip to content

Add GitHub Action summary support to new test framework #46047

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ahus1 merged 1 commit into from
Feb 6, 2026
+196 −14
Merged

Add GitHub Action summary support to new test framework #46047

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ env:
MAVEN_ARGS: "-B -nsu -Daether.connector.http.connectionMaxTtl=25"
SUREFIRE_RERUN_FAILING_COUNT: 2
SUREFIRE_RETRY: "-Dsurefire.rerunFailingTestsCount=2"
KC_TEST_GITHUB_SLOW: "10"

concurrency:
# Only cancel jobs for PR updates
Expand Down
28 changes: 16 additions & 12 deletions ...ework/core/src/main/java/org/keycloak/testframework/KeycloakIntegrationTestExtension.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,51 +19,49 @@

public class KeycloakIntegrationTestExtension implements BeforeAllCallback, BeforeEachCallback, AfterEachCallback, AfterAllCallback, TestWatcher, InvocationInterceptor, ParameterResolver {

private static final LogHandler logHandler = new LogHandler();

@Override
public void beforeAll(ExtensionContext context) {
logHandler.beforeAll(context);
getLogHandler(context).beforeAll(context);
}

@Override
public void beforeEach(ExtensionContext context) {
logHandler.beforeEachStarting(context);
getLogHandler(context).beforeEachStarting(context);
getRegistry(context).beforeEach(context.getRequiredTestInstance(), context.getRequiredTestMethod());
logHandler.beforeEachCompleted(context);
getLogHandler(context).beforeEachCompleted(context);
}

@Override
public void afterEach(ExtensionContext context) {
logHandler.afterEachStarting(context);
getLogHandler(context).afterEachStarting(context);
getRegistry(context).afterEach();
logHandler.afterEachCompleted(context);
getLogHandler(context).afterEachCompleted(context);
}

@Override
public void afterAll(ExtensionContext context) {
logHandler.afterAll(context);
getLogHandler(context).afterAll(context);
getRegistry(context).afterAll();
}

@Override
public void testFailed(ExtensionContext context, Throwable cause) {
logHandler.testFailed(context);
getLogHandler(context).testFailed(context);
}

@Override
public void testDisabled(ExtensionContext context, Optional<String> reason) {
logHandler.testDisabled(context);
getLogHandler(context).testDisabled(context);
}

@Override
public void testSuccessful(ExtensionContext context) {
logHandler.testSuccessful(context);
getLogHandler(context).testSuccessful(context);
}

@Override
public void testAborted(ExtensionContext context, Throwable cause) {
logHandler.testAborted(context);
getLogHandler(context).testAborted(context);
}

@Override
Expand All @@ -78,6 +76,12 @@ public static Registry getRegistry(ExtensionContext context) {
return registry;
}

public static LogHandler getLogHandler(ExtensionContext context) {
ExtensionContext.Store store = context.getRoot().getStore(ExtensionContext.Namespace.GLOBAL);
LogHandler logHandler = (LogHandler) store.computeIfAbsent(LogHandler.class, l -> new LogHandler());
return logHandler;
}

@Override
public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext context) throws ParameterResolutionException {
return getRegistry(context).supportsParameter(parameterContext, context);
Expand Down
17 changes: 15 additions & 2 deletions test-framework/core/src/main/java/org/keycloak/testframework/LogHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import java.util.logging.Handler;

import org.keycloak.testframework.config.Config;
import org.keycloak.testframework.github.GitHubActionReport;

import io.quarkus.runtime.logging.LoggingSetupRecorder;
import io.smallrye.config.SmallRyeConfigProviderResolver;
Expand All @@ -12,10 +13,11 @@
import org.jboss.logmanager.LogManager;
import org.junit.jupiter.api.extension.ExtensionContext;

public class LogHandler {
public class LogHandler implements AutoCloseable {

private static final Logger LOGGER = Logger.getLogger("testinfo");
private final boolean logFilterEnabled;
private final GitHubActionReport gitHubActionReport = new GitHubActionReport();

public LogHandler() {
logFilterEnabled = Config.get("kc.test.log.filter", false, Boolean.class);
Expand Down Expand Up @@ -51,10 +53,15 @@ public void beforeEachStarting(ExtensionContext context) {
public void beforeEachCompleted(ExtensionContext context) {
logTestMethodStatus(context, Status.RUNNING, Logger.Level.DEBUG);
initLogFilter();
gitHubActionReport.onMethodStart();
}

public void afterAll(ExtensionContext context) {
logTestClassStatus(context, Status.FINISHED, Logger.Level.DEBUG);
Status status = context.getExecutionException().isPresent() ? Status.FAILED : Status.SUCCESS;
if (status == Status.FAILED) {
gitHubActionReport.onClassError(context);
}
logTestClassStatus(context, status, Logger.Level.DEBUG);
}

public void afterEachStarting(ExtensionContext context) {
Expand All @@ -65,11 +72,13 @@ public void afterEachCompleted(ExtensionContext context) {
}

public void testSuccessful(ExtensionContext context) {
gitHubActionReport.onMethodSuccess(context);
clearLogFilter(false);
logTestMethodStatus(context, Status.SUCCESS, Logger.Level.DEBUG);
}

public void testFailed(ExtensionContext context) {
gitHubActionReport.onMethodFailed(context);
clearLogFilter(true);
logTestMethodStatus(context, Status.FAILED, Logger.Level.ERROR);
}
Expand All @@ -84,6 +93,10 @@ public void testDisabled(ExtensionContext context) {
logTestMethodStatus(context, Status.DISABLED, Logger.Level.DEBUG);
}

public void close() {
gitHubActionReport.printSummary();
}

private void logDivider(Logger.Level level) {
LOGGER.log(level, "----------------------------------------------------------------");
}
Expand Down
164 changes: 164 additions & 0 deletions test-framework/core/src/main/java/org/keycloak/testframework/github/GitHubActionReport.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
package org.keycloak.testframework.github;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.lang.reflect.Method;
import java.util.Comparator;
import java.util.LinkedList;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;

import org.keycloak.testframework.config.Config;

import org.junit.jupiter.api.extension.ExtensionContext;

public class GitHubActionReport {

private static final String GITHUB_STEP_SUMMARY = System.getenv("GITHUB_STEP_SUMMARY");
private static final String GITHUB_SERVER_URL = System.getenv("GITHUB_SERVER_URL");
private static final String GITHUB_REPOSITORY = System.getenv("GITHUB_REPOSITORY");
private static final String GITHUB_SHA = System.getenv("GITHUB_SHA");

private final boolean enabled;
private final File gitHubStepSummary;
private final String gitRoot = findGitRoot();

private final long slowTestTimeout;

private long testStartedAt;

private List<Failure> failures = new LinkedList<>();
private List<Slow> slowTests = new LinkedList<>();

public GitHubActionReport() {
this.gitHubStepSummary = GITHUB_STEP_SUMMARY != null ? new File(GITHUB_STEP_SUMMARY) : null;
this.enabled = Config.get("kc.test.github.enabled", true, Boolean.class) && gitHubStepSummary != null;
this.slowTestTimeout = TimeUnit.SECONDS.toMillis(Config.get("kc.test.github.slow", 30L, Long.class));
}

public void onClassError(ExtensionContext context) {
if (enabled) {
onError(context, false);
}
}

public void onMethodStart() {
if (enabled && slowTestTimeout >= -1) {
testStartedAt = System.currentTimeMillis();
}
}

public void onMethodSuccess(ExtensionContext context) {
if (enabled) {
if (slowTestTimeout >= -1) {
long executionTime = System.currentTimeMillis() - testStartedAt;
if (executionTime > slowTestTimeout) {
Class<?> testClass = context.getRequiredTestClass();
String file = findJavaClass(testClass);
String link = getLink(file, -1);
slowTests.add(new Slow(context.getRequiredTestClass().getName(), context.getRequiredTestMethod().getName(), executionTime, link));
}
}
}
}

public void onMethodFailed(ExtensionContext context) {
if (enabled) {
onError(context, true);
}
}

public void printSummary() {
if (enabled && (!failures.isEmpty() || !slowTests.isEmpty())) {
try {
PrintWriter printWriter = new PrintWriter(new FileWriter(gitHubStepSummary, true));

if (!failures.isEmpty()) {
printWriter.println("## :x: Failed tests");
printWriter.println("| Test class | Test method | Line | Failure |");
printWriter.println("| ---------- | ----------- | ---- | ------- |");

failures.stream().sorted(Comparator.comparing(Failure::className)).forEach(f ->
printWriter.println("| " + createLink(f.className(), f.link()) + " | " + f.methodName + " | " + f.line() + " | `" + f.message() + "` |")
);
}

if (!slowTests.isEmpty()) {
printWriter.println("## :hourglass: Slow tests detected");
printWriter.println("| Test class | Test method | Execution time (s) |");
printWriter.println("| ---------- | ----------- | -------------- |");

slowTests.stream().sorted(Comparator.comparing(Slow::executionTime).reversed()).forEach(s ->
printWriter.println("| " + createLink(s.className(), s.link()) + " | " + s.methodName() + " | " + (s.executionTime() / 1000) + " |")
);
}

printWriter.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

private void onError(ExtensionContext context, boolean method) {
Optional<Throwable> executionException = context.getExecutionException();
if (executionException.isPresent()) {
Class<?> testClass = context.getRequiredTestClass();
String file = findJavaClass(testClass);

Method testMethod = method ? context.getRequiredTestMethod() : null;
Throwable throwable = executionException.get();
String message = throwable.getMessage();
int line = findLine(testClass, testMethod, throwable);

String link = getLink(file, line);

failures.add(new Failure(testClass.getName(), testMethod != null ? testMethod.getName() : "", message, link, line));
}
}

private String findJavaClass(Class<?> testClass) {
String classFile = testClass.getResource("/" + testClass.getName().replace('.', '/') + ".class").getFile();
return classFile.replace(gitRoot + "/", "").replace("target/test-classes", "src/test/java").replace(".class", ".java");
}

private String getLink(String file, int line) {
String link = GITHUB_SERVER_URL + "/" + GITHUB_REPOSITORY + "/blob/" + GITHUB_SHA + "/" + file;
if (line >= 0) {
link += "#L" + line;
}
return link;
}

private String findGitRoot() {
File file = new File(System.getProperty("user.dir"));
while (file.isDirectory()) {
if (new File(file, ".git").isDirectory()) {
return file.getAbsolutePath();
}
file = file.getParentFile();
}
throw new RuntimeException("Failed to find .git directory");
}

private int findLine(Class<?> testClass, Method testMethod, Throwable throwable) {
for (StackTraceElement stackTraceElement : throwable.getStackTrace()) {
if (stackTraceElement.getClassName().equals(testClass.getName()) && (testMethod == null || stackTraceElement.getMethodName().equals(testMethod.getName()))) {
return stackTraceElement.getLineNumber();
}
}
return -1;
}

private String createLink(String text, String line) {
return "[" + text + "]" + "(" + line + ")";
}

private record Slow(String className, String methodName, long executionTime, String link) {}

private record Failure(String className, String methodName, String message, String link, int line) {}

}
Loading