From 995bbc507d7345699097ad506c158c2725b85c83 Mon Sep 17 00:00:00 2001 From: Ketan Padegaonkar Date: Mon, 1 Dec 2025 17:01:20 +0530 Subject: [PATCH 01/35] chore: add verbose logging flag, move a output statement to debug level --- .../BackwardCompatibilityCheckBaseCommand.kt | 15 +++++++++++++-- .../specmatic/core/TestBackwardCompatibility.kt | 2 +- .../core/TestBackwardCompatibilityKtTest.kt | 9 +++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt index d879b801dd..87e2ecad4f 100644 --- a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt @@ -4,6 +4,7 @@ import io.specmatic.core.IFeature import io.specmatic.core.Results import io.specmatic.core.git.GitCommand import io.specmatic.core.git.SystemGit +import io.specmatic.core.log.Verbose import io.specmatic.core.log.logger import io.specmatic.core.utilities.SystemExit import picocli.CommandLine.Option @@ -45,6 +46,9 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { ) var repoDir: String = "." + @Option(names = ["--debug"], description = ["Write verbose logs to console for debugging"]) + var debugLog = false + abstract fun checkBackwardCompatibility(oldFeature: IFeature, newFeature: IFeature): Results abstract fun File.isValidFileFormat(): Boolean abstract fun File.isValidSpec(): Boolean @@ -59,7 +63,11 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { open fun getUnusedExamples(feature: IFeature): Set = emptySet() final override fun call() { + if(debugLog) + logger = Verbose() + gitCommand = SystemGit(workingDirectory = Paths.get(repoDir).absolutePathString()) + addShutdownHook() val filteredSpecs = getChangedSpecs() val result = try { @@ -130,7 +138,8 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { val queue = ArrayDeque(specFiles) while (queue.isNotEmpty()) { - val combinedPattern = Pattern.compile(queue.toSet().joinToString(prefix = "\\b(?:", separator = "|", postfix = ")\\b") { specFile -> + val combinedPattern = Pattern.compile( + queue.toSet().joinToString(prefix = "\\b(?:", separator = "|", postfix = ")\\b") { specFile -> regexForMatchingReferred(specFile.name).let { Regex.escape(it) } }) @@ -266,7 +275,9 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { logger.log(backwardCompatibilityResult.report().prependIndent(TWO_INDENTS)) logVerdictFor( specFilePath, - "(INCOMPATIBLE) The changes to the spec are NOT backward compatible with the corresponding spec from ${baseBranch()}".prependIndent(ONE_INDENT), + "(INCOMPATIBLE) The changes to the spec are NOT backward compatible with the corresponding spec from ${baseBranch()}".prependIndent( + ONE_INDENT + ), ) val compatibilityResult = diff --git a/core/src/main/kotlin/io/specmatic/core/TestBackwardCompatibility.kt b/core/src/main/kotlin/io/specmatic/core/TestBackwardCompatibility.kt index 783384d497..a5fa933624 100644 --- a/core/src/main/kotlin/io/specmatic/core/TestBackwardCompatibility.kt +++ b/core/src/main/kotlin/io/specmatic/core/TestBackwardCompatibility.kt @@ -15,7 +15,7 @@ fun testBackwardCompatibility(older: Feature, newer: Feature): Results { .fold(Results() to emptySet()) { (results, olderScenariosTested), olderScenario -> val olderScenarioDescription = olderScenario.testDescription() if (olderScenarioDescription !in olderScenariosTested) { - logger.log("[Compatibility Check] ${olderScenarioDescription.trim()}") + logger.debug("[Compatibility Check] ${olderScenarioDescription.trim()}") logger.boundary() } diff --git a/core/src/test/kotlin/io/specmatic/core/TestBackwardCompatibilityKtTest.kt b/core/src/test/kotlin/io/specmatic/core/TestBackwardCompatibilityKtTest.kt index 57caf07226..6e657c0f57 100644 --- a/core/src/test/kotlin/io/specmatic/core/TestBackwardCompatibilityKtTest.kt +++ b/core/src/test/kotlin/io/specmatic/core/TestBackwardCompatibilityKtTest.kt @@ -1,6 +1,8 @@ package io.specmatic.core import io.specmatic.conversions.OpenApiSpecification +import io.specmatic.core.log.Verbose +import io.specmatic.core.log.logger import io.specmatic.stub.captureStandardOutput import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test @@ -2788,6 +2790,9 @@ paths: @Test fun `should show a message when testing each API`() { + val oldLogger = logger + try { + val feature: Feature = """ openapi: 3.0.0 @@ -2804,11 +2809,15 @@ paths: description: Created """.trimIndent().openAPIToContract() + logger = Verbose() val (stdout, _) = captureStandardOutput { testBackwardCompatibility(feature, feature) } assertThat(stdout).contains("[Compatibility Check] Scenario: POST /products -> 201") + } finally { + logger = oldLogger + } } @Test From 11ab8fc87676b904e906060d4e104a9613ba0ec1 Mon Sep 17 00:00:00 2001 From: vedubhat Date: Mon, 1 Dec 2025 05:42:17 -0800 Subject: [PATCH 02/35] refactor : move central_contract_repo_report command to specmatic-reporter --- .../CentralContractRepoReportCommand.kt | 42 -- .../kotlin/application/SpecmaticCommand.kt | 1 - ...CentralContractRepoReportCommandTestE2E.kt | 152 ------- .../kotlin/application/StubCommandTest.kt | 6 + core/build.gradle.kts | 4 - .../AsyncAPICentralContractRepoReportUtils.kt | 117 ------ .../reports/CentralContractRepoReport.kt | 97 ----- .../reports/CentralContractRepoReportJson.kt | 33 -- .../reports/CentralContractRepoReportTest.kt | 374 ------------------ 9 files changed, 6 insertions(+), 820 deletions(-) delete mode 100644 application/src/main/kotlin/application/CentralContractRepoReportCommand.kt delete mode 100644 application/src/test/kotlin/application/CentralContractRepoReportCommandTestE2E.kt delete mode 100644 core/src/main/kotlin/io/specmatic/reports/AsyncAPICentralContractRepoReportUtils.kt delete mode 100644 core/src/main/kotlin/io/specmatic/reports/CentralContractRepoReport.kt delete mode 100644 core/src/main/kotlin/io/specmatic/reports/CentralContractRepoReportJson.kt delete mode 100644 core/src/test/kotlin/reports/CentralContractRepoReportTest.kt diff --git a/application/src/main/kotlin/application/CentralContractRepoReportCommand.kt b/application/src/main/kotlin/application/CentralContractRepoReportCommand.kt deleted file mode 100644 index 50e125f9e8..0000000000 --- a/application/src/main/kotlin/application/CentralContractRepoReportCommand.kt +++ /dev/null @@ -1,42 +0,0 @@ -package application - -import io.specmatic.core.log.logger -import io.specmatic.core.utilities.saveJsonFile -import io.specmatic.license.core.cli.Category -import io.specmatic.reports.CentralContractRepoReport -import kotlinx.serialization.encodeToString -import kotlinx.serialization.json.Json -import picocli.CommandLine -import java.util.concurrent.Callable - -@CommandLine.Command( - name = "central-contract-repo-report", - mixinStandardHelpOptions = true, - description = ["Generate the Central Contract Repo Report"] -) -@Category("Specmatic core") -class CentralContractRepoReportCommand : Callable { - - companion object { - const val REPORT_PATH = "./build/reports/specmatic" - const val REPORT_FILE_NAME = "central_contract_repo_report.json" - } - - @CommandLine.Option(names = ["--baseDir"], description = ["Directory to treated as the root for API specifications"], defaultValue = "") - lateinit var baseDir: String - - override fun call() { - val report = CentralContractRepoReport().generate(baseDir) - if(report.specifications.isEmpty()) { - logger.log("No specifications found, hence the Central Contract Repo Report has not been generated.") - } - else { - logger.log("Saving Central Contract Repo Report json to $REPORT_PATH ...") - val json = Json { - encodeDefaults = false - } - val reportJson = json.encodeToString(report) - saveJsonFile(reportJson, REPORT_PATH, REPORT_FILE_NAME) - } - } -} diff --git a/application/src/main/kotlin/application/SpecmaticCommand.kt b/application/src/main/kotlin/application/SpecmaticCommand.kt index 203441cabd..2333718e78 100644 --- a/application/src/main/kotlin/application/SpecmaticCommand.kt +++ b/application/src/main/kotlin/application/SpecmaticCommand.kt @@ -32,7 +32,6 @@ object SpecmaticCoreSubcommands : CliConfigurer { ExamplesCommand(), StubCommand(), TestCommand(), - CentralContractRepoReportCommand(), ConfigCommand(), McpBaseCommand(), *ReporterSubcommands.subcommands(), diff --git a/application/src/test/kotlin/application/CentralContractRepoReportCommandTestE2E.kt b/application/src/test/kotlin/application/CentralContractRepoReportCommandTestE2E.kt deleted file mode 100644 index 681e2102e0..0000000000 --- a/application/src/test/kotlin/application/CentralContractRepoReportCommandTestE2E.kt +++ /dev/null @@ -1,152 +0,0 @@ -package application - -import io.specmatic.reports.CentralContractRepoReportJson -import io.specmatic.reports.OpenAPISpecificationOperation -import io.specmatic.reports.SpecificationOperation -import io.specmatic.reports.SpecificationRow -import kotlinx.serialization.json.Json -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.AfterAll -import org.junit.jupiter.api.BeforeAll -import org.junit.jupiter.api.Test -import java.io.File - -class CentralContractRepoReportCommandTestE2E { - - private val centralContractRepoReportCommand = CentralContractRepoReportCommand() - - @Test - fun `test generates report json file`() { - centralContractRepoReportCommand.baseDir = "" - centralContractRepoReportCommand.call() - val reportJson: CentralContractRepoReportJson = Json.decodeFromString(reportFile.readText()) - - val expectedSpecificationRow = SpecificationRow( - osAgnosticPath("specifications/service1/service1.yaml"), - "HTTP", - "OPENAPI", - listOf( - OpenAPISpecificationOperation( - "/hello/{id}", - "GET", - 200 - ), - OpenAPISpecificationOperation( - "/hello/{id}", - "GET", - 404 - ), - OpenAPISpecificationOperation( - "/hello/{id}", - "GET", - 400 - ) - ) - ) - - assertThat(reportJson.specifications.contains(expectedSpecificationRow)).isTrue() - } - - @Test - fun `report only contains specifications within baseDir`() { - createSpecFiles("./specifications/service2/service2.yaml") - centralContractRepoReportCommand.baseDir = "specifications/service2" - centralContractRepoReportCommand.call() - val reportJson: CentralContractRepoReportJson = Json.decodeFromString(reportFile.readText()) - - val expectedSpecificationRow = SpecificationRow( - osAgnosticPath("specifications/service2/service2.yaml"), - "HTTP", - "OPENAPI", - listOf( - OpenAPISpecificationOperation( - "/hello/{id}", - "GET", - 200 - ), - OpenAPISpecificationOperation( - "/hello/{id}", - "GET", - 404 - ), - OpenAPISpecificationOperation( - "/hello/{id}", - "GET", - 400 - ) - ) - ) - - assertThat(reportJson.specifications).containsOnly(expectedSpecificationRow) - } - - companion object { - private val reportFile = File(osAgnosticPath("./build/reports/specmatic/central_contract_repo_report.json")) - - @JvmStatic - @BeforeAll - fun setupBeforeAll() { - createSpecFiles("./specifications/service1/service1.yaml") - } - - @JvmStatic - @AfterAll - fun tearDownAfterAll() { - File(osAgnosticPath("./specifications")).deleteRecursively() - reportFile.delete() - } - - private fun createSpecFiles(specFilePath: String) { - val service1spec = """ -openapi: 3.0.0 -info: - title: Sample API - description: Optional multiline or single-line description in [CommonMark](http://commonmark.org/help/) or HTML. - version: 0.1.9 -servers: - - url: http://api.example.com/v1 - description: Optional server description, e.g. Main (production) server - - url: http://staging-api.example.com - description: Optional server description, e.g. Internal staging server for testing -paths: - /hello/{id}: - get: - summary: hello world - description: Optional extended description in CommonMark or HTML. - parameters: - - in: path - name: id - schema: - type: integer - required: true - description: Numeric ID - responses: - '200': - description: Says hello - content: - application/json: - schema: - type: string - '404': - description: Not Found - content: - application/json: - schema: - type: string - '400': - description: Bad Request - content: - application/json: - schema: - type: string - """ - val service1File = File(osAgnosticPath(specFilePath)) - service1File.parentFile.mkdirs() - service1File.writeText(service1spec) - } - } -} - -fun osAgnosticPath(path: String): String { - return path.replace("/", File.separator) -} diff --git a/application/src/test/kotlin/application/StubCommandTest.kt b/application/src/test/kotlin/application/StubCommandTest.kt index 057c54f426..9a5c3fdb99 100644 --- a/application/src/test/kotlin/application/StubCommandTest.kt +++ b/application/src/test/kotlin/application/StubCommandTest.kt @@ -71,6 +71,7 @@ internal class StubCommandTest { verify(exactly = 0) { specmaticConfig.contractStubPathData() } } + @Test fun `should attempt to start a HTTP stub`(@TempDir tempDir: File) { val contractPath = osAgnosticPath("${tempDir.path}/contract.$CONTRACT_EXTENSION") @@ -283,4 +284,9 @@ internal class StubCommandTest { System.clearProperty(Flags.SPECMATIC_BASE_URL) } } + + fun osAgnosticPath(path: String): String { + return path.replace("/", File.separator) + } } + diff --git a/core/build.gradle.kts b/core/build.gradle.kts index e56e807d7b..81131d6d3c 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -53,11 +53,7 @@ dependencies { testImplementation("org.assertj:assertj-core:3.27.6") testImplementation("io.ktor:ktor-client-mock-jvm:2.3.13") implementation("org.thymeleaf:thymeleaf:3.1.3.RELEASE") - implementation("org.junit.platform:junit-platform-launcher:1.13.4") - - // TEMPORARILY ADDED FOR GENERATING THE CENTRAL CONTRACT REPO - implementation("com.asyncapi:asyncapi-core:1.0.0-RC4") } configurations.implementation.configure { diff --git a/core/src/main/kotlin/io/specmatic/reports/AsyncAPICentralContractRepoReportUtils.kt b/core/src/main/kotlin/io/specmatic/reports/AsyncAPICentralContractRepoReportUtils.kt deleted file mode 100644 index 402beebd5d..0000000000 --- a/core/src/main/kotlin/io/specmatic/reports/AsyncAPICentralContractRepoReportUtils.kt +++ /dev/null @@ -1,117 +0,0 @@ -package io.specmatic.reports - -import com.asyncapi.schemas.asyncapi.Reference -import com.asyncapi.v3._0_0.model.operation.reply.OperationReply -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import io.specmatic.core.log.logger -import io.specmatic.core.utilities.exceptionCauseMessage -import org.yaml.snakeyaml.Yaml -import java.io.File -import java.nio.file.Paths -import com.asyncapi.v2._6_0.model.AsyncAPI as AsyncAPI_2_6_0 -import com.asyncapi.v3._0_0.model.AsyncAPI as AsyncAPI_3_0_0 - -fun getAsyncAPISpecificationRows( - specifications: List, - currentWorkingDir: String -): List { - return specifications.map { file -> - SpecificationRow( - specification = file.relativeTo(File("").canonicalFile).path, - serviceType = SpecType.ASYNCAPI.name, - specType = SpecType.ASYNCAPI.name, - operations = asyncAPIOperationsFrom(file) - ) - } -} - -fun hasAsyncApiFileExtension(specPath: String): Boolean { - return specPath.endsWith(".yaml") || - specPath.endsWith(".yml") || - specPath.endsWith(".json") -} - -fun isAsyncAPI(specPath: String): Boolean { - return try { - Yaml().load>(File(specPath).reader()).contains("asyncapi") - } catch(e: Throwable) { - logger.log(e, "Could not parse $specPath") - false - } -} - -private fun asyncAPIOperationsFrom(file: File): List { - return when { - asyncAPIVersionOf(file) == "3.0.0" -> asyncAPIV3OperationsFrom(file) - asyncAPIVersionOf(file).startsWith("2.") -> asyncAPIV2OperationsFrom(file) - else -> emptyList() - } -} - -private fun asyncAPIV3OperationsFrom(file: File): List { - val asyncAPI = try { - ObjectMapper(YAMLFactory()).readValue( - file.readText(), - AsyncAPI_3_0_0::class.java - ) - } catch (e: Exception) { - logger.log("Could not parse ${file.path} due to the following error:") - logger.log(exceptionCauseMessage(e)) - null - } - if(asyncAPI == null) return emptyList() - - return asyncAPI.operations.orEmpty().flatMap { (name, op) -> - op as com.asyncapi.v3._0_0.model.operation.Operation - val opBasedOperation = AsyncAPISpecificationOperation( - operation = name, - channel = op.channel.name(), - action = op.action.name.lowercase() - ) - val replyOp = (op.reply as? OperationReply).takeIf { op.reply != null } - val replyOpBasedOperation = AsyncAPISpecificationOperation( - operation = name, - channel = replyOp?.channel?.name().orEmpty(), - action = "send" - ).takeIf { replyOp != null && replyOp.channel != null } - - listOfNotNull(opBasedOperation, replyOpBasedOperation) - } -} - -private fun asyncAPIV2OperationsFrom(file: File): List { - val asyncAPI = try { - ObjectMapper(YAMLFactory()).readValue( - file.readText(), - AsyncAPI_2_6_0::class.java - ) - } catch (e: Exception) { - logger.log("Could not parse ${file.path} due to the following error:") - logger.log(exceptionCauseMessage(e)) - null - } - if(asyncAPI == null) return emptyList() - - return asyncAPI.channels.map { (channelName, channel) -> - channel.publish?.let { op -> - AsyncAPISpecificationOperation( - operation = op.operationId.orEmpty(), - channel = channelName, - action = "receive" - ) - } ?: channel.subscribe?.let { op -> - AsyncAPISpecificationOperation( - operation = op.operationId.orEmpty(), - channel = channelName, - action = "send" - ) - } - }.filterNotNull() -} - -private fun asyncAPIVersionOf(file: File): String = ObjectMapper(YAMLFactory()).readTree(file).let { - it.findPath("asyncapi").asText() -} - -private fun Reference.name(): String = ref.substringAfterLast("/") diff --git a/core/src/main/kotlin/io/specmatic/reports/CentralContractRepoReport.kt b/core/src/main/kotlin/io/specmatic/reports/CentralContractRepoReport.kt deleted file mode 100644 index 232bbce323..0000000000 --- a/core/src/main/kotlin/io/specmatic/reports/CentralContractRepoReport.kt +++ /dev/null @@ -1,97 +0,0 @@ -package io.specmatic.reports - -import io.specmatic.conversions.OpenApiSpecification -import io.specmatic.conversions.convertPathParameterStyle -import io.specmatic.core.log.logger -import io.specmatic.core.utilities.exceptionCauseMessage -import io.specmatic.stub.hasOpenApiFileExtension -import io.specmatic.stub.isOpenAPI -import java.io.File - -class CentralContractRepoReport { - fun generate(currentWorkingDir: String = ""): CentralContractRepoReportJson { - val searchPath = File(currentWorkingDir).canonicalPath - logger.log("Searching for specification files at: $searchPath") - val specifications = findSpecifications(searchPath) - - val openAPISpecificationRows = getOpenAPISpecificationRows( - specifications[SpecType.OPENAPI].orEmpty(), - searchPath - ) - val asyncAPISpecificationRows = getAsyncAPISpecificationRows( - specifications[SpecType.ASYNCAPI].orEmpty(), - searchPath - ) - return CentralContractRepoReportJson( - openAPISpecificationRows + asyncAPISpecificationRows - ) - } - - private fun getOpenAPISpecificationRows( - specifications: List, - currentWorkingDir: String - ): List { - val currentWorkingDirPath = File(currentWorkingDir).absoluteFile - - return specifications.mapNotNull { - try { - val feature = OpenApiSpecification.fromYAML(it.readText(), it.path).toFeature() - Pair(it, feature) - } - catch (e:Throwable){ - logger.log("Could not parse ${it.path} due to the following error:") - logger.log(exceptionCauseMessage(e)) - null - } - } - .filter { (spec, feature) -> - if (feature.scenarios.isEmpty()) { - logger.log("Excluding specification: ${spec.path} as it does not have any paths ") - } - feature.scenarios.isNotEmpty() - }.map { (spec, feature) -> - SpecificationRow( - spec.relativeTo(File("").canonicalFile).path, - feature.serviceType, - SpecType.OPENAPI.name, - feature.scenarios.map { - OpenAPISpecificationOperation( - convertPathParameterStyle(it.path), - it.method, - it.httpResponsePattern.status - ) - } - ) - } - } - - private fun findSpecifications(currentDirectoryPath: String): Map> { - val currentDirectory = File(currentDirectoryPath) - val openApiSpecifications = mutableListOf() - val asyncApiSpecifications = mutableListOf() - val allFiles = currentDirectory.listFiles() ?: emptyArray() - for (file in allFiles) { - if (file.isDirectory) { - val specMap = findSpecifications(file.canonicalPath) - openApiSpecifications.addAll(specMap[SpecType.OPENAPI].orEmpty()) - asyncApiSpecifications.addAll(specMap[SpecType.ASYNCAPI].orEmpty()) - } else { - if (hasOpenApiFileExtension(file.canonicalPath) && isOpenAPI(file.canonicalPath)) { - openApiSpecifications.add(file) - } - if (hasAsyncApiFileExtension(file.canonicalPath) && isAsyncAPI(file.canonicalPath)) { - asyncApiSpecifications.add(file) - } - } - } - return mapOf( - SpecType.OPENAPI to openApiSpecifications, - SpecType.ASYNCAPI to asyncApiSpecifications - ) - } -} - -enum class SpecType { - OPENAPI, - ASYNCAPI -} diff --git a/core/src/main/kotlin/io/specmatic/reports/CentralContractRepoReportJson.kt b/core/src/main/kotlin/io/specmatic/reports/CentralContractRepoReportJson.kt deleted file mode 100644 index 4d4890e902..0000000000 --- a/core/src/main/kotlin/io/specmatic/reports/CentralContractRepoReportJson.kt +++ /dev/null @@ -1,33 +0,0 @@ -package io.specmatic.reports - -import kotlinx.serialization.Serializable - -@Serializable -data class CentralContractRepoReportJson( - val specifications: List -) - -@Serializable -data class SpecificationRow( - val specification: String, - val serviceType: String?, - val specType: String?, - val operations: List -) - -@Serializable -sealed interface SpecificationOperation - -@Serializable -data class OpenAPISpecificationOperation( - val path: String, - val method: String, - val responseCode: Int -): SpecificationOperation - -@Serializable -data class AsyncAPISpecificationOperation( - val operation: String, - val channel: String, - val action: String -) : SpecificationOperation \ No newline at end of file diff --git a/core/src/test/kotlin/reports/CentralContractRepoReportTest.kt b/core/src/test/kotlin/reports/CentralContractRepoReportTest.kt deleted file mode 100644 index 5dc2df96c5..0000000000 --- a/core/src/test/kotlin/reports/CentralContractRepoReportTest.kt +++ /dev/null @@ -1,374 +0,0 @@ -package reports - -import io.specmatic.osAgnosticPath -import io.specmatic.reports.CentralContractRepoReport -import io.specmatic.reports.CentralContractRepoReportJson -import io.specmatic.reports.OpenAPISpecificationOperation -import io.specmatic.reports.SpecificationRow -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.AfterAll -import org.junit.jupiter.api.BeforeAll -import org.junit.jupiter.api.Test -import java.io.File - -class CentralContractRepoReportTest { - - @Test - fun `test generates report based on all the open api specifications present in the specified dir`() { - val report = CentralContractRepoReport().generate("./specifications/service1") - assertThat(osAgnosticPaths(report)).isEqualTo( - osAgnosticPaths( - CentralContractRepoReportJson( - listOf( - SpecificationRow( - "specifications/service1/service1.yaml", - "HTTP", - "OPENAPI", - listOf( - OpenAPISpecificationOperation( - "/hello/{id}", - "GET", - 200 - ), - OpenAPISpecificationOperation( - "/hello/{id}", - "GET", - 404 - ), - OpenAPISpecificationOperation( - "/hello/{id}", - "GET", - 400 - ) - ) - ) - ) - ) - ) - ) - } - - @Test - fun `test generates report based on asyncapi 3_0_0 spec with operations section`() { - val report = CentralContractRepoReport().generate("./specifications/asyncapi3spec") - assertThat(osAgnosticPaths(report)).isEqualTo( - osAgnosticPaths( - CentralContractRepoReportJson( - listOf( - SpecificationRow( - "specifications/asyncapi3spec/asyncapi3spec.yaml", - "ASYNCAPI", - "ASYNCAPI", - listOf( - io.specmatic.reports.AsyncAPISpecificationOperation( - operation = "placeOrder", - channel = "NewOrderPlaced", - action = "receive" - ), - io.specmatic.reports.AsyncAPISpecificationOperation( - operation = "placeOrder", - channel = "OrderInitiated", - action = "send" - ), - io.specmatic.reports.AsyncAPISpecificationOperation( - operation = "cancelOrder", - channel = "OrderCancellationRequested", - action = "receive" - ), - io.specmatic.reports.AsyncAPISpecificationOperation( - operation = "cancelOrder", - channel = "OrderCancelled", - action = "send" - ) - ) - ) - ) - ) - ) - ) - } - - @Test - fun `test generates report based on asyncapi 2 6 0 spec with publish and subscribe operations`() { - val report = CentralContractRepoReport().generate("./specifications/asyncapi2spec") - assertThat(osAgnosticPaths(report)).isEqualTo( - osAgnosticPaths( - CentralContractRepoReportJson( - listOf( - SpecificationRow( - "specifications/asyncapi2spec/asyncapi2spec.yaml", - "ASYNCAPI", - "ASYNCAPI", - listOf( - io.specmatic.reports.AsyncAPISpecificationOperation( - operation = "placeOrder", - channel = "place-order", - action = "receive" - ), - io.specmatic.reports.AsyncAPISpecificationOperation( - operation = "processOrder", - channel = "process-order", - action = "send" - ) - ) - ) - ) - ) - ) - ) - } - - private fun osAgnosticPaths(report: CentralContractRepoReportJson): CentralContractRepoReportJson { - return report.copy( - specifications = report.specifications.map { - it.copy( - specification = osAgnosticPath(it.specification) - ) - } - ) - } - - companion object { - - @JvmStatic - @BeforeAll - fun setupBeforeAll() { - createSpecFiles() - } - - @JvmStatic - @AfterAll - fun tearDownAfterAll() { - File("./specifications").deleteRecursively() - } - - private fun createSpecFiles() { - val service1spec = """ -openapi: 3.0.0 -info: - title: Sample API - description: Optional multiline or single-line description in [CommonMark](http://commonmark.org/help/) or HTML. - version: 0.1.9 -servers: - - url: http://api.example.com/v1 - description: Optional server description, e.g. Main (production) server - - url: http://staging-api.example.com - description: Optional server description, e.g. Internal staging server for testing -paths: - /hello/{id}: - get: - summary: hello world - description: Optional extended description in CommonMark or HTML. - parameters: - - in: path - name: id - schema: - type: integer - required: true - description: Numeric ID - responses: - '200': - description: Says hello - content: - application/json: - schema: - type: string - '404': - description: Not Found - content: - application/json: - schema: - type: string - '400': - description: Bad Request - content: - application/json: - schema: - type: string - """ - val service1File = File("./specifications/service1/service1.yaml") - service1File.parentFile.mkdirs() - service1File.writeText(service1spec) - - - val service2spec= """ -openapi: 3.0.0 -info: - title: Order API - version: '1.0' -servers: - - url: 'http://localhost:3000' -paths: - '/products/{id}': - parameters: - - schema: - type: number - name: id - in: path - required: true - examples: - GET_DETAILS_10: - value: 10 - GET_DETAILS_20: - value: 20 - get: - summary: Fetch product details - tags: [] - responses: - '200': - description: OK - content: - application/json: - schema: - ${'$'}ref: './common.yaml#/components/schemas/Product' - examples: - GET_DETAILS_10: - value: - name: 'XYZ Phone' - type: 'gadget' - inventory: 10 - id: 10 - GET_DETAILS_20: - value: - name: 'Macbook' - type: 'gadget' - inventory: 10 - id: 20 - """.trimIndent() - - val service2File = File("./specifications/service2/service2.yaml") - service2File.parentFile.mkdirs() - service2File.writeText(service2spec) - - val commonSpec = """ - openapi: 3.0.0 - info: - title: Common schema - version: '1.0' - paths: {} - components: - schemas: - ProductDetails: - title: Product Details - type: object - properties: - name: - type: string - type: - ${'$'}ref: '#/components/schemas/ProductType' - inventory: - type: integer - required: - - name - - type - - inventory - ProductType: - type: string - title: Product Type - enum: - - book - - food - - gadget - - other - ProductId: - title: Product Id - type: object - properties: - id: - type: integer - required: - - id - Product: - title: Product - allOf: - - ${'$'}ref: '#/components/schemas/ProductId' - - ${'$'}ref: '#/components/schemas/ProductDetails' - """.trimIndent() - - val commonFile = File("./specifications/service2/common.yaml") - commonFile.writeText(commonSpec) - - createAsyncAPI3_0_0Spec() - createAsyncAPI2_6_0Spec() - } - - private fun createAsyncAPI3_0_0Spec() { - val asyncapi3Spec = """ - asyncapi: 3.0.0 - info: - title: Order API - version: 1.0.0 - channels: - NewOrderPlaced: - address: new-orders - messages: - placeOrder.message: - ${'$'}ref: '#/components/messages/OrderRequest' - OrderInitiated: - address: wip-orders - messages: - processOrder.message: - ${'$'}ref: '#/components/messages/Order' - OrderCancellationRequested: - address: to-be-cancelled-orders - messages: - cancelOrder.message: - ${'$'}ref: '#/components/messages/CancelOrderRequest' - OrderCancelled: - address: cancelled-orders - messages: - processCancellation.message: - ${'$'}ref: '#/components/messages/CancellationReference' - operations: - placeOrder: - action: receive - channel: - ${'$'}ref: '#/channels/NewOrderPlaced' - messages: - - ${'$'}ref: '#/channels/NewOrderPlaced/messages/placeOrder.message' - reply: - channel: - ${'$'}ref: '#/channels/OrderInitiated' - messages: - - ${'$'}ref: '#/channels/OrderInitiated/messages/processOrder.message' - cancelOrder: - action: receive - channel: - ${'$'}ref: '#/channels/OrderCancellationRequested' - messages: - - ${'$'}ref: '#/channels/OrderCancellationRequested/messages/cancelOrder.message' - reply: - channel: - ${'$'}ref: '#/channels/OrderCancelled' - messages: - - ${'$'}ref: '#/channels/OrderCancelled/messages/processCancellation.message' - """ - val asyncapi3File = File("./specifications/asyncapi3spec/asyncapi3spec.yaml") - asyncapi3File.parentFile.mkdirs() - asyncapi3File.writeText(asyncapi3Spec) - } - - private fun createAsyncAPI2_6_0Spec() { - val asyncapi2Spec = """ - asyncapi: 2.6.0 - info: - title: Order API - version: '1.0.0' - channels: - place-order: - publish: - operationId: placeOrder - message: - ${'$'}ref: "#/components/messages/OrderRequest" - process-order: - subscribe: - operationId: processOrder - message: - ${'$'}ref: "#/components/messages/Order" - """ - val asyncapi2File = File("./specifications/asyncapi2spec/asyncapi2spec.yaml") - asyncapi2File.parentFile.mkdirs() - asyncapi2File.writeText(asyncapi2Spec) - } - } -} \ No newline at end of file From 23bc8ba41e2e68e37cc08b72708ab8a3b51e4234 Mon Sep 17 00:00:00 2001 From: vedubhat Date: Mon, 1 Dec 2025 22:24:09 -0800 Subject: [PATCH 03/35] chore: update specmatic-reporter version to 0.1.7 --- application/src/test/kotlin/application/StubCommandTest.kt | 1 + gradle.properties | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/test/kotlin/application/StubCommandTest.kt b/application/src/test/kotlin/application/StubCommandTest.kt index 9a5c3fdb99..354f6e2b42 100644 --- a/application/src/test/kotlin/application/StubCommandTest.kt +++ b/application/src/test/kotlin/application/StubCommandTest.kt @@ -21,6 +21,7 @@ import picocli.CommandLine import java.io.File import java.nio.file.Path + internal class StubCommandTest { @MockK diff --git a/gradle.properties b/gradle.properties index ef03669fbd..1496043966 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ group=io.specmatic version=2.32.1-SNAPSHOT specmaticGradlePluginVersion=0.13.8 -specmaticReporterVersion=0.1.6 +specmaticReporterVersion=0.1.7 kotlin.daemon.jvmargs=-Xmx1024m org.gradle.jvmargs=-Xmx1024m From 302261538f990ea6c7b782e2c2c24c1e44a85bd8 Mon Sep 17 00:00:00 2001 From: Specmatic GitHub Service Account Date: Tue, 2 Dec 2025 06:45:33 +0000 Subject: [PATCH 04/35] chore(release): post-release bump version 2.32.2-SNAPSHOT --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 28a6c7da62..31e2aa5b23 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ group=io.specmatic -version=2.32.1 +version=2.32.2-SNAPSHOT specmaticGradlePluginVersion=0.13.8 specmaticReporterVersion=0.1.6 kotlin.daemon.jvmargs=-Xmx1024m From fb9a707667cfc28b476fcedfc4d4286073b82a23 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Thu, 27 Nov 2025 10:30:05 +0530 Subject: [PATCH 05/35] feat: use enhanced ctrf reporter from specmatic-reporter --- .../kotlin/io/specmatic/test/ContractTest.kt | 12 ++- .../io/specmatic/test/ScenarioAsTest.kt | 64 ++++++++++---- .../test/ScenarioTestGenerationException.kt | 17 ++-- .../test/ScenarioTestGenerationFailure.kt | 17 ++-- .../io/specmatic/test/TestResultRecord.kt | 86 ++++++++++++++----- .../specmatic/test/SpecmaticAfterAllHook.kt | 9 +- .../specmatic/test/SpecmaticJUnitSupport.kt | 26 +++++- .../coverage/OpenApiCoverageReportInput.kt | 61 +++++-------- 8 files changed, 194 insertions(+), 98 deletions(-) diff --git a/core/src/main/kotlin/io/specmatic/test/ContractTest.kt b/core/src/main/kotlin/io/specmatic/test/ContractTest.kt index fbf1e5f943..97541cbd2b 100644 --- a/core/src/main/kotlin/io/specmatic/test/ContractTest.kt +++ b/core/src/main/kotlin/io/specmatic/test/ContractTest.kt @@ -17,10 +17,16 @@ interface ResponseValidator { } interface ContractTest : HasScenarioMetadata { - fun testResultRecord(result: Result, response: HttpResponse?): TestResultRecord? + fun testResultRecord(executionResult: ContractTestExecutionResult): TestResultRecord? fun testDescription(): String - fun runTest(testBaseURL: String, timeoutInMilliseconds: Long): Pair - fun runTest(testExecutor: TestExecutor): Pair + fun runTest(testBaseURL: String, timeoutInMilliseconds: Long): ContractTestExecutionResult + fun runTest(testExecutor: TestExecutor): ContractTestExecutionResult fun plusValidator(validator: ResponseValidator): ContractTest } + +data class ContractTestExecutionResult( + val result: Result, + val request: HttpRequest? = null, + val response: HttpResponse? = null +) diff --git a/core/src/main/kotlin/io/specmatic/test/ScenarioAsTest.kt b/core/src/main/kotlin/io/specmatic/test/ScenarioAsTest.kt index 6229d419b6..4ed3eca7aa 100644 --- a/core/src/main/kotlin/io/specmatic/test/ScenarioAsTest.kt +++ b/core/src/main/kotlin/io/specmatic/test/ScenarioAsTest.kt @@ -1,7 +1,15 @@ package io.specmatic.test import io.specmatic.conversions.convertPathParameterStyle -import io.specmatic.core.* +import io.specmatic.core.ContractAndResponseMismatch +import io.specmatic.core.Feature +import io.specmatic.core.FlagsBased +import io.specmatic.core.HttpRequest +import io.specmatic.core.HttpResponse +import io.specmatic.core.Result +import io.specmatic.core.Scenario +import io.specmatic.core.ValidateUnexpectedKeys +import io.specmatic.core.Workflow import io.specmatic.core.log.HttpLogMessage import io.specmatic.core.log.LogMessage import io.specmatic.core.log.logger @@ -11,7 +19,6 @@ import io.specmatic.stub.SPECMATIC_RESPONSE_CODE_HEADER import io.specmatic.test.handlers.ResponseHandler import io.specmatic.test.handlers.ResponseHandlerRegistry import io.specmatic.test.handlers.ResponseHandlingResult -import java.time.Duration import java.time.Instant data class ScenarioAsTest( @@ -39,7 +46,8 @@ data class ScenarioAsTest( override fun toScenarioMetadata() = scenario.toScenarioMetadata() - override fun testResultRecord(result: Result, response: HttpResponse?): TestResultRecord { + override fun testResultRecord(executionResult: ContractTestExecutionResult): TestResultRecord { + val (result, request, response) = executionResult val resultStatus = result.testResult() return TestResultRecord( @@ -47,17 +55,20 @@ data class ScenarioAsTest( method = scenario.method, requestContentType = scenario.requestContentType, responseStatus = scenario.status, + request = request, + response = response, result = resultStatus, sourceProvider = sourceProvider, - sourceRepository = sourceRepository, - sourceRepositoryBranch = sourceRepositoryBranch, + repository = sourceRepository, + branch = sourceRepositoryBranch, specification = specification, serviceType = serviceType, actualResponseStatus = response?.status ?: 0, scenarioResult = result, soapAction = scenario.httpRequestPattern.getSOAPAction().takeIf { scenario.isGherkinScenario }, isGherkin = scenario.isGherkinScenario, - duration = Duration.between(startTime, Instant.now()).toMillis() + requestTime = startTime, + responseTime = Instant.now() ) } @@ -65,7 +76,7 @@ data class ScenarioAsTest( return scenario.testDescription() } - override fun runTest(testBaseURL: String, timeoutInMilliseconds: Long): Pair { + override fun runTest(testBaseURL: String, timeoutInMilliseconds: Long): ContractTestExecutionResult { val log: (LogMessage) -> Unit = { logMessage -> logger.log(logMessage.withComment(this.annotations)) } @@ -75,7 +86,7 @@ data class ScenarioAsTest( return runTest(httpClient) } - override fun runTest(testExecutor: TestExecutor): Pair { + override fun runTest(testExecutor: TestExecutor): ContractTestExecutionResult { startTime = Instant.now() val newExecutor = if (testExecutor is HttpClient) { val log: (LogMessage) -> Unit = { logMessage -> @@ -87,9 +98,9 @@ data class ScenarioAsTest( testExecutor } - val (result, response) = executeTestAndReturnResultAndResponse(scenario, newExecutor, flagsBased) + val executionResult = executeTestAndReturnResultAndResponse(scenario, newExecutor, flagsBased) endTime = Instant.now() - return Pair(result.updateScenario(scenario), response) + return executionResult.copy(result = executionResult.result.updateScenario(scenario)) } override fun plusValidator(validator: ResponseValidator): ScenarioAsTest { @@ -102,7 +113,7 @@ data class ScenarioAsTest( testScenario: Scenario, testExecutor: TestExecutor, flagsBased: FlagsBased - ): Pair { + ): ContractTestExecutionResult { try { val request = testScenario.generateHttpRequest(flagsBased).let { workflow.updateRequest(it, originalScenario).adjustPayloadForContentType() @@ -116,13 +127,21 @@ data class ScenarioAsTest( workflow.extractDataFrom(response, originalScenario) val validatorResult = validators.asSequence().map { it.validate(scenario, response) }.filterNotNull().firstOrNull() if (validatorResult is Result.Failure) { - return Pair(validatorResult.withBindings(testScenario.bindings, response), response) + return ContractTestExecutionResult( + result = validatorResult.withBindings(testScenario.bindings, response), + request = request, + response = response + ) } val testResult = validatorResult ?: testResult(request, response, testScenario, flagsBased) val responseHandler = response.getResponseHandlerIfExists() if (testResult is Result.Failure && responseHandler == null) { - return Pair(testResult.withBindings(testScenario.bindings, response), response) + return ContractTestExecutionResult( + result = testResult.withBindings(testScenario.bindings, response), + request = request, + response = response + ) } val responseToCheckAndStore = when (responseHandler) { @@ -133,7 +152,11 @@ data class ScenarioAsTest( is ResponseHandlingResult.Continue -> handlerResult.response is ResponseHandlingResult.Stop -> { val bindingResponse = handlerResult.response ?: response - return Pair(handlerResult.result.withBindings(testScenario.bindings, bindingResponse), bindingResponse) + return ContractTestExecutionResult( + result = handlerResult.result.withBindings(testScenario.bindings, bindingResponse), + request = request, + response = bindingResponse + ) } } } @@ -144,11 +167,16 @@ data class ScenarioAsTest( }.firstOrNull() ?: Result.Success() testScenario.exampleRow?.let { ExampleProcessor.store(it, request, responseToCheckAndStore) } - return Pair(result.withBindings(testScenario.bindings, response), response) + + return ContractTestExecutionResult( + result = result.withBindings(testScenario.bindings, response), + request = request, + response = response + ) } catch (exception: Throwable) { - return Pair( - Result.Failure(exceptionCauseMessage(exception)) - .also { failure -> failure.updateScenario(testScenario) }, null + return ContractTestExecutionResult( + result = Result.Failure(exceptionCauseMessage(exception)) + .also { failure -> failure.updateScenario(testScenario) } ) } } diff --git a/core/src/main/kotlin/io/specmatic/test/ScenarioTestGenerationException.kt b/core/src/main/kotlin/io/specmatic/test/ScenarioTestGenerationException.kt index 3fd7867273..c1094ebee8 100644 --- a/core/src/main/kotlin/io/specmatic/test/ScenarioTestGenerationException.kt +++ b/core/src/main/kotlin/io/specmatic/test/ScenarioTestGenerationException.kt @@ -29,16 +29,19 @@ class ScenarioTestGenerationException( override fun toScenarioMetadata() = scenario.toScenarioMetadata() - override fun testResultRecord(result: Result, response: HttpResponse?): TestResultRecord { + override fun testResultRecord(executionResult: ContractTestExecutionResult): TestResultRecord { + val (result, request, response) = executionResult return TestResultRecord( path = convertPathParameterStyle(scenario.path), method = scenario.method, requestContentType = scenario.requestContentType, responseStatus = scenario.status, + request = request, + response = response, result = result.testResult(), sourceProvider = scenario.sourceProvider, - sourceRepository = scenario.sourceRepository, - sourceRepositoryBranch = scenario.sourceRepositoryBranch, + repository = scenario.sourceRepository, + branch = scenario.sourceRepositoryBranch, specification = scenario.specification, serviceType = scenario.serviceType, actualResponseStatus = 0, @@ -52,13 +55,13 @@ class ScenarioTestGenerationException( return scenario.testDescription() } - override fun runTest(testBaseURL: String, timeoutInMilliseconds: Long): Pair { + override fun runTest(testBaseURL: String, timeoutInMilliseconds: Long): ContractTestExecutionResult { val log: (LogMessage) -> Unit = { logMessage -> logger.log(logMessage) } val httpClient = LegacyHttpClient(testBaseURL, log = log, timeoutInMilliseconds = timeoutInMilliseconds) return runTest(httpClient) } - override fun runTest(testExecutor: TestExecutor): Pair { + override fun runTest(testExecutor: TestExecutor): ContractTestExecutionResult { testExecutor.preExecuteScenario(scenario, httpRequest) return error() } @@ -67,12 +70,12 @@ class ScenarioTestGenerationException( return this } - fun error(): Pair { + fun error(): ContractTestExecutionResult { val result: Result = when(e) { is ContractException -> Result.Failure(errorMessage, e.failure(), breadCrumb = breadCrumb ?: "").updateScenario(scenario) else -> Result.Failure(errorMessage + " - " + exceptionCauseMessage(e), breadCrumb = breadCrumb ?: "").updateScenario(scenario) } - return Pair(result, null) + return ContractTestExecutionResult(result = result) } } diff --git a/core/src/main/kotlin/io/specmatic/test/ScenarioTestGenerationFailure.kt b/core/src/main/kotlin/io/specmatic/test/ScenarioTestGenerationFailure.kt index 59f1d493dc..e1cba0d336 100644 --- a/core/src/main/kotlin/io/specmatic/test/ScenarioTestGenerationFailure.kt +++ b/core/src/main/kotlin/io/specmatic/test/ScenarioTestGenerationFailure.kt @@ -26,16 +26,19 @@ class ScenarioTestGenerationFailure( override fun toScenarioMetadata() = scenario.toScenarioMetadata() - override fun testResultRecord(result: Result, response: HttpResponse?): TestResultRecord { + override fun testResultRecord(executionResult: ContractTestExecutionResult): TestResultRecord { + val (result, request, response) = executionResult return TestResultRecord( path = convertPathParameterStyle(scenario.path), method = scenario.method, requestContentType = scenario.requestContentType, responseStatus = scenario.status, + request = request, + response = response, result = result.testResult(), sourceProvider = scenario.sourceProvider, - sourceRepository = scenario.sourceRepository, - sourceRepositoryBranch = scenario.sourceRepositoryBranch, + repository = scenario.sourceRepository, + branch = scenario.sourceRepositoryBranch, specification = scenario.specification, serviceType = scenario.serviceType, actualResponseStatus = 0, @@ -49,15 +52,17 @@ class ScenarioTestGenerationFailure( return scenario.testDescription() } - override fun runTest(testBaseURL: String, timeoutInMilliseconds: Long): Pair { + override fun runTest(testBaseURL: String, timeoutInMilliseconds: Long): ContractTestExecutionResult { val log: (LogMessage) -> Unit = { logMessage -> logger.log(logMessage) } val httpClient = LegacyHttpClient(testBaseURL, log = log, timeoutInMilliseconds = timeoutInMilliseconds) return runTest(httpClient) } - override fun runTest(testExecutor: TestExecutor): Pair { + override fun runTest(testExecutor: TestExecutor): ContractTestExecutionResult { testExecutor.preExecuteScenario(scenario, httpRequest) - return Pair(failureCause.updateScenario(scenario), null) + return ContractTestExecutionResult( + result = failureCause.updateScenario(scenario) + ) } override fun plusValidator(validator: ResponseValidator): ContractTest { diff --git a/core/src/main/kotlin/io/specmatic/test/TestResultRecord.kt b/core/src/main/kotlin/io/specmatic/test/TestResultRecord.kt index 6c74da1463..0b3ce82f5e 100644 --- a/core/src/main/kotlin/io/specmatic/test/TestResultRecord.kt +++ b/core/src/main/kotlin/io/specmatic/test/TestResultRecord.kt @@ -1,16 +1,28 @@ package io.specmatic.test +import io.specmatic.core.HttpRequest +import io.specmatic.core.HttpResponse import io.specmatic.core.Result +import io.specmatic.core.pattern.ContractException +import io.specmatic.reporter.ctrf.model.CtrfTestMetadata +import io.specmatic.reporter.ctrf.model.CtrfTestResultRecord +import io.specmatic.reporter.ctrf.model.operation.OpenAPICtrfOperation +import io.specmatic.reporter.internal.dto.coverage.CoverageStatus +import io.specmatic.reporter.internal.dto.ctrf.CtrfOperation import io.specmatic.reporter.model.TestResult +import java.time.Duration +import java.time.Instant data class TestResultRecord( val path: String, val method: String, val responseStatus: Int, + val request: HttpRequest?, + val response: HttpResponse?, override val result: TestResult, val sourceProvider: String? = null, - override val sourceRepository: String? = null, - override val sourceRepositoryBranch: String? = null, + override val repository: String? = null, + override val branch: String? = null, override val specification: String? = null, val serviceType: String? = null, val actualResponseStatus: Int = 0, @@ -20,30 +32,33 @@ data class TestResultRecord( val requestContentType: String? = null, val soapAction: String? = null, val isGherkin: Boolean = false, - override val duration: Long = 0, + val requestTime: Instant? = null, + val responseTime: Instant? = null, + override val duration: Long = durationFrom(requestTime, responseTime), override val rawStatus: String? = result.toString(), - override val testType: String = "ContractTest" -): io.specmatic.reporter.ctrf.model.TestResultRecord { + override val testType: String = "ContractTest", + override val operation: CtrfOperation = OpenAPICtrfOperation( + path = path, + method = method, + contentType = requestContentType.orEmpty(), + responseCode = actualResponseStatus + ) +): CtrfTestResultRecord { val isExercised = result !in setOf(TestResult.MissingInSpec, TestResult.NotCovered) val isCovered = result !in setOf(TestResult.MissingInSpec, TestResult.NotCovered) fun isConnectionRefused() = actualResponseStatus == 0 - override fun extraFields(): Map { - val extra = mutableMapOf() - - extra["httpMethod"] = method - extra["httpPath"] = path - requestContentType?.let { extra["requestContentType"] = it } - extra["expectedStatus"] = responseStatus - extra["actualStatus"] = actualResponseStatus - - sourceRepository?.let { extra["sourceRepository"] = it } - sourceRepositoryBranch?.let { extra["sourceBranch"] = it } - specification?.let { extra["specification"] = it } - - extra["isWip"] = isWip - return extra + override fun extraFields(): CtrfTestMetadata { + return CtrfTestMetadata( + valid = isValid, + isWip = isWip, + input = request?.toLogString().orEmpty(), + output = response?.toLogString().orEmpty(), + inputTime = requestTime?.toEpochMilli() ?: 0L, + outputTime = responseTime?.toEpochMilli() ?: 0L, + operation = operation + ) } override fun tags(): List { @@ -82,4 +97,35 @@ data class TestResultRecord( else -> "${method.uppercase()} $path (${responseStatus})" } } + + companion object { + fun List.getCoverageStatus(): CoverageStatus { + if(this.any { it.isWip }) return CoverageStatus.WIP + + if(!this.any { it.isValid }) { + return when (this.first().result) { + TestResult.MissingInSpec -> CoverageStatus.MISSING_IN_SPEC + else -> CoverageStatus.INVALID + } + } + + if (this.any { it.isExercised }) { + return when (this.first().result) { + TestResult.NotImplemented -> CoverageStatus.NOT_IMPLEMENTED + else -> CoverageStatus.COVERED + } + } + + return when (val result = this.first().result) { + TestResult.NotCovered -> CoverageStatus.NOT_COVERED + TestResult.MissingInSpec -> CoverageStatus.MISSING_IN_SPEC + else -> throw ContractException("Cannot determine remarks for unknown test result: $result") + } + } + } } + +private fun durationFrom(requestTime: Instant?, responseTime: Instant?) = + if (requestTime != null && responseTime != null) + Duration.between(requestTime, responseTime).toMillis() + else 0L diff --git a/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticAfterAllHook.kt b/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticAfterAllHook.kt index 73d52ecf78..a90cd9d1d4 100644 --- a/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticAfterAllHook.kt +++ b/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticAfterAllHook.kt @@ -1,6 +1,13 @@ package io.specmatic.test +import io.specmatic.reporter.ctrf.model.CtrfSpecConfig interface SpecmaticAfterAllHook { - fun onAfterAllTests(testResultRecords: List?, startTime: Long, endTime: Long, coverage: Int) + fun onAfterAllTests( + testResultRecords: List?, + startTime: Long, + endTime: Long, + coverage: Int, + specConfigs: List + ) } \ No newline at end of file diff --git a/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt b/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt index 909a0e1a7d..a52ac963c5 100644 --- a/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt +++ b/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt @@ -21,6 +21,7 @@ import io.specmatic.core.utilities.Flags.Companion.getLongValue import io.specmatic.core.value.JSONArrayValue import io.specmatic.core.value.JSONObjectValue import io.specmatic.core.value.Value +import io.specmatic.reporter.ctrf.model.CtrfSpecConfig import io.specmatic.stub.hasOpenApiFileExtension import io.specmatic.stub.isOpenAPI import io.specmatic.test.reports.OpenApiCoverageReportProcessor @@ -191,12 +192,30 @@ open class SpecmaticJUnitSupport { val report = openApiCoverageReportInput.generateCoverageReport(emptyList()) val start = startTime?.toEpochMilli() ?: 0L val end = startTime?.let { Instant.now().toEpochMilli() } ?: 0L + + val specConfigs = openApiCoverageReportInput.endpoints() + .groupBy { + it.specification.orEmpty() + }.flatMap { (_, groupedEndpoints) -> + groupedEndpoints.map { + CtrfSpecConfig( + serviceType = it.serviceType.orEmpty(), + specType = "OPENAPI", + specification = it.specification.orEmpty(), + sourceProvider = it.sourceProvider, + repository = it.sourceRepository, + branch = it.sourceRepositoryBranch, + serverUrl = "" + ) + } + } hooks.forEach { it.onAfterAllTests( testResultRecords = report.testResultRecords, coverage = report.totalCoveragePercentage, startTime = start, endTime = end, + specConfigs = specConfigs ) } } @@ -421,7 +440,7 @@ open class SpecmaticJUnitSupport { DynamicTest.dynamicTest(contractTest.testDescription()) { threads.add(Thread.currentThread().name) - var testResult: Pair? = null + var testResult: ContractTestExecutionResult? = null try { val log: (LogMessage) -> Unit = { logMessage -> @@ -462,9 +481,8 @@ open class SpecmaticJUnitSupport { throw e } finally { if (testResult != null) { - val (result, response) = testResult - contractTest.testResultRecord(result, response)?.let { testREsultRecord -> - openApiCoverageReportInput.addTestReportRecords(testREsultRecord) + contractTest.testResultRecord(testResult)?.let { testResultRecord -> + openApiCoverageReportInput.addTestReportRecords(testResultRecord) } } } diff --git a/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/OpenApiCoverageReportInput.kt b/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/OpenApiCoverageReportInput.kt index 36cb27ba56..c12774472c 100644 --- a/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/OpenApiCoverageReportInput.kt +++ b/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/OpenApiCoverageReportInput.kt @@ -13,6 +13,7 @@ import io.specmatic.reporter.model.TestResult import io.specmatic.test.API import io.specmatic.test.HttpInteractionsLog import io.specmatic.test.TestResultRecord +import io.specmatic.test.TestResultRecord.Companion.getCoverageStatus import io.specmatic.test.reports.TestReportListener import io.specmatic.test.reports.coverage.console.GroupedTestResultRecords import io.specmatic.test.reports.coverage.console.OpenAPICoverageConsoleReport @@ -35,6 +36,8 @@ class OpenApiCoverageReportInput( private val previousTestResultRecord: List = emptyList(), private val filteredEndpoints: MutableList = mutableListOf(), ) { + fun endpoints() = allEndpoints.toList() + fun totalDuration(): Long { return httpInteractionsLog.totalDuration() } @@ -161,15 +164,17 @@ class OpenApiCoverageReportInput( return this.plus( endpointsWithoutTests.map { endpoint -> TestResultRecord( - endpoint.path, - endpoint.method, - endpoint.responseStatus, - TestResult.NotCovered, - endpoint.sourceProvider, - endpoint.sourceRepository, - endpoint.sourceRepositoryBranch, - endpoint.specification, - endpoint.serviceType + path = endpoint.path, + method = endpoint.method, + responseStatus = endpoint.responseStatus, + request = null, + response = null, + result = TestResult.NotCovered, + sourceProvider = endpoint.sourceProvider, + repository = endpoint.sourceRepository, + branch = endpoint.sourceRepositoryBranch, + specification = endpoint.specification, + serviceType = endpoint.serviceType ) } ) @@ -189,8 +194,8 @@ class OpenApiCoverageReportInput( return this.groupBy { CoverageGroupKey( it.sourceProvider, - it.sourceRepository, - it.sourceRepositoryBranch, + it.repository, + it.branch, it.specification, it.serviceType ) @@ -219,30 +224,6 @@ class OpenApiCoverageReportInput( } } - private fun List.getCoverageStatus(): CoverageStatus { - if(this.any { it.isWip }) return CoverageStatus.WIP - - if(!this.any { it.isValid }) { - return when (this.first().result) { - TestResult.MissingInSpec -> CoverageStatus.MISSING_IN_SPEC - else -> CoverageStatus.INVALID - } - } - - if (this.any { it.isExercised }) { - return when (this.first().result) { - TestResult.NotImplemented -> CoverageStatus.NOT_IMPLEMENTED - else -> CoverageStatus.COVERED - } - } - - return when (val result = this.first().result) { - TestResult.NotCovered -> CoverageStatus.NOT_COVERED - TestResult.MissingInSpec -> CoverageStatus.MISSING_IN_SPEC - else -> throw ContractException("Cannot determine remarks for unknown test result: $result") - } - } - private fun List.groupRecords(): GroupedTestResultRecords { return groupBy { it.path }.mapValues { (_, pathMap) -> pathMap.groupBy { it.soapAction ?: it.method }.mapValues { (_, methodMap) -> @@ -263,10 +244,12 @@ class OpenApiCoverageReportInput( noTestResultFoundForThisAPI && isNotExcluded }.map { api -> TestResultRecord( - api.path, - api.method, - 0, - TestResult.MissingInSpec, + path = api.path, + method = api.method, + responseStatus = 0, + request = null, + response = null, + result = TestResult.MissingInSpec, serviceType = SERVICE_TYPE_HTTP ) } From 00a88f479a2cdfdbb2be19b9dcd002bb90b18db8 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Mon, 1 Dec 2025 15:28:50 +0530 Subject: [PATCH 06/35] refactor: update the signature of TestResultRecord --- .../main/kotlin/io/specmatic/test/TestResultRecord.kt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/core/src/main/kotlin/io/specmatic/test/TestResultRecord.kt b/core/src/main/kotlin/io/specmatic/test/TestResultRecord.kt index 0b3ce82f5e..3455116899 100644 --- a/core/src/main/kotlin/io/specmatic/test/TestResultRecord.kt +++ b/core/src/main/kotlin/io/specmatic/test/TestResultRecord.kt @@ -6,10 +6,10 @@ import io.specmatic.core.Result import io.specmatic.core.pattern.ContractException import io.specmatic.reporter.ctrf.model.CtrfTestMetadata import io.specmatic.reporter.ctrf.model.CtrfTestResultRecord -import io.specmatic.reporter.ctrf.model.operation.OpenAPICtrfOperation import io.specmatic.reporter.internal.dto.coverage.CoverageStatus -import io.specmatic.reporter.internal.dto.ctrf.CtrfOperation +import io.specmatic.reporter.internal.dto.spec.operation.APIOperation import io.specmatic.reporter.model.TestResult +import io.specmatic.reporter.spec.model.OpenAPIOperation import java.time.Duration import java.time.Instant @@ -37,7 +37,7 @@ data class TestResultRecord( override val duration: Long = durationFrom(requestTime, responseTime), override val rawStatus: String? = result.toString(), override val testType: String = "ContractTest", - override val operation: CtrfOperation = OpenAPICtrfOperation( + override val operation: APIOperation = OpenAPIOperation( path = path, method = method, contentType = requestContentType.orEmpty(), @@ -56,8 +56,7 @@ data class TestResultRecord( input = request?.toLogString().orEmpty(), output = response?.toLogString().orEmpty(), inputTime = requestTime?.toEpochMilli() ?: 0L, - outputTime = responseTime?.toEpochMilli() ?: 0L, - operation = operation + outputTime = responseTime?.toEpochMilli() ?: 0L ) } From 0b9f768c281da94128683a348d82e1cf9869af1c Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Mon, 1 Dec 2025 17:13:23 +0530 Subject: [PATCH 07/35] chore: fix tests --- .../test/ApiCoverageReportInputTest.kt | 72 +++++++++---------- .../test/ApiCoverageReportStatusTest.kt | 16 ++--- .../specmatic/test/ApiCoverageReportTest.kt | 66 ++++++++++------- .../OpenApiCoverageReportInputTest.kt | 24 ++++++- 4 files changed, 106 insertions(+), 72 deletions(-) diff --git a/junit5-support/src/test/kotlin/io/specmatic/test/ApiCoverageReportInputTest.kt b/junit5-support/src/test/kotlin/io/specmatic/test/ApiCoverageReportInputTest.kt index 3b16fa0311..7844cc64df 100644 --- a/junit5-support/src/test/kotlin/io/specmatic/test/ApiCoverageReportInputTest.kt +++ b/junit5-support/src/test/kotlin/io/specmatic/test/ApiCoverageReportInputTest.kt @@ -24,10 +24,10 @@ class ApiCoverageReportInputTest { @Test fun `test generates api coverage report when all endpoints are covered`() { val testReportRecords = mutableListOf( - TestResultRecord("/route1", "GET", 200, TestResult.Success), - TestResultRecord("/route1", "POST", 200, TestResult.Success), - TestResultRecord("/route1", "POST", 401, TestResult.Success), - TestResultRecord("/route2", "GET", 200, TestResult.Success), + TestResultRecord("/route1", "GET", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route1", "POST", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route1", "POST", 401, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route2", "GET", 200, request = null, response = null, result = TestResult.Success), ) val applicationAPIs = mutableListOf( API("GET", "/route1"), @@ -70,10 +70,10 @@ class ApiCoverageReportInputTest { ) val testReportRecords = mutableListOf( - TestResultRecord("/route1", "GET", 200, TestResult.Success), - TestResultRecord("/route1", "POST", 200, TestResult.Success), - TestResultRecord("/route1", "POST", 401, TestResult.Success), - TestResultRecord("/route2", "GET", 200, TestResult.Success) + TestResultRecord("/route1", "GET", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route1", "POST", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route1", "POST", 401, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route2", "GET", 200, request = null, response = null, result = TestResult.Success) ) val endpointsInSpec = mutableListOf( @@ -105,11 +105,11 @@ class ApiCoverageReportInputTest { @Test fun `test generates api coverage report when some endpoints are marked as excluded`() { val testReportRecords = mutableListOf( - TestResultRecord("/route1", "GET", 200, TestResult.Success), - TestResultRecord("/route1", "POST", 200, TestResult.Success), - TestResultRecord("/route1", "POST", 401, TestResult.Success), - TestResultRecord("/route2", "GET", 200, TestResult.Success), - TestResultRecord("/route2", "POST", 200, TestResult.Success) + TestResultRecord("/route1", "GET", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route1", "POST", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route1", "POST", 401, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route2", "GET", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route2", "POST", 200, request = null, response = null, result = TestResult.Success) ) val applicationAPIs = mutableListOf( API("GET", "/route1"), @@ -154,11 +154,11 @@ class ApiCoverageReportInputTest { @Test fun `test generates empty api coverage report when all endpoints are marked as excluded`() { val testReportRecords = mutableListOf( - TestResultRecord("/route1", "GET", 200, TestResult.Success), - TestResultRecord("/route1", "POST", 200, TestResult.Success), - TestResultRecord("/route1", "POST", 401, TestResult.Success), - TestResultRecord("/route2", "GET", 200, TestResult.Success), - TestResultRecord("/route2", "POST", 200, TestResult.Success) + TestResultRecord("/route1", "GET", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route1", "POST", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route1", "POST", 401, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route2", "GET", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route2", "POST", 200, request = null, response = null, result = TestResult.Success) ) val applicationAPIs = mutableListOf( API("GET", "/route1"), @@ -208,10 +208,10 @@ class ApiCoverageReportInputTest { ) val testReportRecords = mutableListOf( - TestResultRecord("/route1", "GET", 200, TestResult.Success), - TestResultRecord("/route1", "POST", 200, TestResult.Success), - TestResultRecord("/route2", "GET", 200, TestResult.Failed, actualResponseStatus = 404), - TestResultRecord("/route2", "POST", 200, TestResult.Failed, actualResponseStatus = 404) + TestResultRecord("/route1", "GET", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route1", "POST", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route2", "GET", 200, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404), + TestResultRecord("/route2", "POST", 200, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404) ) val endpointsInSpec = mutableListOf( @@ -250,10 +250,10 @@ class ApiCoverageReportInputTest { ) val testReportRecords = mutableListOf( - TestResultRecord("/route1", "GET", 200, TestResult.Success), - TestResultRecord("/route1", "POST", 200, TestResult.Success), - TestResultRecord("/route2", "GET", 200, TestResult.Success), - TestResultRecord("/route2", "POST", 200, TestResult.Failed, actualResponseStatus = 404) + TestResultRecord("/route1", "GET", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route1", "POST", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route2", "GET", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route2", "POST", 200, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404) ) val endpointsInSpec = mutableListOf( @@ -293,10 +293,10 @@ class ApiCoverageReportInputTest { ) val testReportRecords = mutableListOf( - TestResultRecord("/route1", "GET", 200, TestResult.Success, "git", "https://github.com/specmatic/specmatic-order-contracts.git", "main", "in/specmatic/examples/store/route1.yaml", "HTTP"), - TestResultRecord("/route1", "POST", 200, TestResult.Success, "git", "https://github.com/specmatic/specmatic-order-contracts.git", "main", "in/specmatic/examples/store/route1.yaml", "HTTP"), - TestResultRecord("/route2", "GET", 200, TestResult.Success, "git", "https://github.com/specmatic/specmatic-order-contracts.git", "main", "in/specmatic/examples/store/route2.yaml", "HTTP"), - TestResultRecord("/route2", "POST", 200, TestResult.Failed, "git", "https://github.com/specmatic/specmatic-order-contracts.git", "main", "in/specmatic/examples/store/route2.yaml", "HTTP", actualResponseStatus = 404) + TestResultRecord("/route1", "GET", 200, request = null, response = null, result = TestResult.Success, "git", "https://github.com/specmatic/specmatic-order-contracts.git", "main", "in/specmatic/examples/store/route1.yaml", "HTTP"), + TestResultRecord("/route1", "POST", 200, request = null, response = null, result = TestResult.Success, "git", "https://github.com/specmatic/specmatic-order-contracts.git", "main", "in/specmatic/examples/store/route1.yaml", "HTTP"), + TestResultRecord("/route2", "GET", 200, request = null, response = null, result = TestResult.Success, "git", "https://github.com/specmatic/specmatic-order-contracts.git", "main", "in/specmatic/examples/store/route2.yaml", "HTTP"), + TestResultRecord("/route2", "POST", 200, request = null, response = null, result = TestResult.Failed, "git", "https://github.com/specmatic/specmatic-order-contracts.git", "main", "in/specmatic/examples/store/route2.yaml", "HTTP", actualResponseStatus = 404) ) val endpointsInSpec = mutableListOf( @@ -316,12 +316,12 @@ class ApiCoverageReportInputTest { @Test fun `test generates api coverage report with endpoints present in spec but not tested`() { val testReportRecords = mutableListOf( - TestResultRecord("/route1", "GET", 200, TestResult.Success), - TestResultRecord("/route1", "POST", 200, TestResult.Success), - TestResultRecord("/route1", "POST", 401, TestResult.Success), - TestResultRecord("/route2", "GET", 200, TestResult.Success), - TestResultRecord("/route2", "GET", 404, TestResult.Success), - TestResultRecord("/route2", "POST", 500, TestResult.Success), + TestResultRecord("/route1", "GET", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route1", "POST", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route1", "POST", 401, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route2", "GET", 200, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route2", "GET", 404, request = null, response = null, result = TestResult.Success), + TestResultRecord("/route2", "POST", 500, request = null, response = null, result = TestResult.Success), ) val applicationAPIs = mutableListOf( API("GET", "/route1"), diff --git a/junit5-support/src/test/kotlin/io/specmatic/test/ApiCoverageReportStatusTest.kt b/junit5-support/src/test/kotlin/io/specmatic/test/ApiCoverageReportStatusTest.kt index 6119ff93f5..1bed33ebe2 100644 --- a/junit5-support/src/test/kotlin/io/specmatic/test/ApiCoverageReportStatusTest.kt +++ b/junit5-support/src/test/kotlin/io/specmatic/test/ApiCoverageReportStatusTest.kt @@ -25,7 +25,7 @@ class ApiCoverageReportStatusTest { ) val contractTestResults = mutableListOf( - TestResultRecord("/route1", "GET", 200, TestResult.Success), + TestResultRecord("/route1", "GET", 200, request = null, response = null, result = TestResult.Success), ) val apiCoverageReport = OpenApiCoverageReportInput( @@ -51,7 +51,7 @@ class ApiCoverageReportStatusTest { val applicationAPIs = mutableListOf() val contractTestResults = mutableListOf( - TestResultRecord("/route1", "GET", 200, TestResult.Success), + TestResultRecord("/route1", "GET", 200, request = null, response = null, result = TestResult.Success), ) val apiCoverageReport = OpenApiCoverageReportInput( @@ -79,7 +79,7 @@ class ApiCoverageReportStatusTest { ) val contractTestResults = mutableListOf( - TestResultRecord("/route1", "GET", 200, TestResult.Failed, actualResponseStatus = 400) + TestResultRecord("/route1", "GET", 200, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 400) ) val apiCoverageReport = OpenApiCoverageReportInput( @@ -106,7 +106,7 @@ class ApiCoverageReportStatusTest { val applicationAPIs = mutableListOf() val contractTestResults = mutableListOf( - TestResultRecord("/route1", "GET", 200, TestResult.Failed, actualResponseStatus = 400), + TestResultRecord("/route1", "GET", 200, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 400), ) val apiCoverageReport = OpenApiCoverageReportInput( @@ -136,8 +136,8 @@ class ApiCoverageReportStatusTest { ) val contractTestResults = mutableListOf( - TestResultRecord("/route1", "GET", 200, TestResult.Success, actualResponseStatus = 200), - TestResultRecord("/route2", "GET", 200, TestResult.Failed, actualResponseStatus = 404), + TestResultRecord("/route1", "GET", 200, request = null, response = null, result = TestResult.Success, actualResponseStatus = 200), + TestResultRecord("/route2", "GET", 200, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404), ) val apiCoverageReport = OpenApiCoverageReportInput( @@ -168,7 +168,7 @@ class ApiCoverageReportStatusTest { ) val contractTestResults = mutableListOf( - TestResultRecord("/route1", "GET", 200, TestResult.Success) + TestResultRecord("/route1", "GET", 200, request = null, response = null, result = TestResult.Success) ) val apiCoverageReport = OpenApiCoverageReportInput( @@ -198,7 +198,7 @@ class ApiCoverageReportStatusTest { ) val contractTestResults = mutableListOf( - TestResultRecord("/route1", "GET", 200, TestResult.Success) + TestResultRecord("/route1", "GET", 200, request = null, response = null, result = TestResult.Success) ) val apiCoverageReport = OpenApiCoverageReportInput( diff --git a/junit5-support/src/test/kotlin/io/specmatic/test/ApiCoverageReportTest.kt b/junit5-support/src/test/kotlin/io/specmatic/test/ApiCoverageReportTest.kt index d270a598d1..0382603262 100644 --- a/junit5-support/src/test/kotlin/io/specmatic/test/ApiCoverageReportTest.kt +++ b/junit5-support/src/test/kotlin/io/specmatic/test/ApiCoverageReportTest.kt @@ -42,7 +42,7 @@ class ApiCoverageReportTest { ) val applicationAPIs = mutableListOf() val testResultRecords = mutableListOf( - TestResultRecord("/order/{id}", "GET", 200, TestResult.Failed, actualResponseStatus = 404) + TestResultRecord("/order/{id}", "GET", 200, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec, applicationAPIs) @@ -65,8 +65,24 @@ class ApiCoverageReportTest { ) val applicationAPIs = mutableListOf() val testResultRecords = mutableListOf( - TestResultRecord("/order/{id}", "POST", 201, TestResult.Failed, actualResponseStatus = 404), - TestResultRecord("/order/{id}", "POST", 400, TestResult.Failed, actualResponseStatus = 404) + TestResultRecord( + "/order/{id}", + "POST", + 201, + request = null, + response = null, + result = TestResult.Failed, + actualResponseStatus = 404 + ), + TestResultRecord( + "/order/{id}", + "POST", + 400, + request = null, + response = null, + result = TestResult.Failed, + actualResponseStatus = 404 + ) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec, applicationAPIs) @@ -88,7 +104,7 @@ class ApiCoverageReportTest { fun `GET 200 in spec not implemented without actuator`() { val endpointsInSpec = mutableListOf(Endpoint("/order/{id}", "GET", 200)) val testResultRecords = - mutableListOf(TestResultRecord("/order/{id}", "GET", 200, TestResult.Failed, actualResponseStatus = 404)) + mutableListOf(TestResultRecord("/order/{id}", "GET", 200, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404)) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec) assertThat(apiCoverageReport).isEqualTo( @@ -109,8 +125,8 @@ class ApiCoverageReportTest { Endpoint("/order/{id}", "POST", 201), Endpoint("/order/{id}", "POST", 400) ) val testResultRecords = mutableListOf( - TestResultRecord("/order/{id}", "POST", 201, TestResult.Failed, actualResponseStatus = 404), - TestResultRecord("/order/{id}", "POST", 400, TestResult.Failed, actualResponseStatus = 404) + TestResultRecord("/order/{id}", "POST", 201, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404), + TestResultRecord("/order/{id}", "POST", 400, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec) @@ -132,7 +148,7 @@ class ApiCoverageReportTest { ) val applicationAPIs = mutableListOf() val testResultRecords = mutableListOf( - TestResultRecord("/order/{id}", "GET", 200, TestResult.Failed, actualResponseStatus = 404) + TestResultRecord("/order/{id}", "GET", 200, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec, applicationAPIs, filteredEndpoints = endpointsInSpec) @@ -155,8 +171,8 @@ class ApiCoverageReportTest { ) val applicationAPIS = mutableListOf() val testResultRecords = mutableListOf( - TestResultRecord("/order/{id}", "POST", 201, TestResult.Failed, actualResponseStatus = 404), - TestResultRecord("/order/{id}", "POST", 400, TestResult.Failed, actualResponseStatus = 404) + TestResultRecord("/order/{id}", "POST", 201, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404), + TestResultRecord("/order/{id}", "POST", 400, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec, applicationAPIS, filteredEndpoints = endpointsInSpec) @@ -178,7 +194,7 @@ class ApiCoverageReportTest { Endpoint("/order/{id}", "GET", 200), Endpoint("/order/{id}", "GET", 404) ) val testResultRecords = mutableListOf( - TestResultRecord("/order/{id}", "GET", 200, TestResult.Failed, actualResponseStatus = 404) + TestResultRecord("/order/{id}", "GET", 200, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec, filteredEndpoints = endpointsInSpec) @@ -200,8 +216,8 @@ class ApiCoverageReportTest { Endpoint("/order/{id}", "POST", 404) ) val testResultRecords = mutableListOf( - TestResultRecord("/order/{id}", "POST", 201, TestResult.Failed, actualResponseStatus = 404), - TestResultRecord("/order/{id}", "POST", 400, TestResult.Failed, actualResponseStatus = 404) + TestResultRecord("/order/{id}", "POST", 201, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404), + TestResultRecord("/order/{id}", "POST", 400, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec, filteredEndpoints = endpointsInSpec) @@ -225,7 +241,7 @@ class ApiCoverageReportTest { API("GET", "/order/{id}") ) val testResultRecords = mutableListOf( - TestResultRecord("/order/{id}", "GET", 200, TestResult.Success, actualResponseStatus = 200) + TestResultRecord("/order/{id}", "GET", 200, request = null, response = null, result = TestResult.Success, actualResponseStatus = 200) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec, applicationAPIs, filteredEndpoints = endpointsInSpec) @@ -251,8 +267,8 @@ class ApiCoverageReportTest { API("POST", "/order/{id}") ) val testResultRecords = mutableListOf( - TestResultRecord("/order/{id}", "POST", 201, TestResult.Success, actualResponseStatus = 201), - TestResultRecord("/order/{id}", "POST", 400, TestResult.Success, actualResponseStatus = 400) + TestResultRecord("/order/{id}", "POST", 201, request = null, response = null, result = TestResult.Success, actualResponseStatus = 201), + TestResultRecord("/order/{id}", "POST", 400, request = null, response = null, result = TestResult.Success, actualResponseStatus = 400) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec, applicationAPIs, filteredEndpoints = endpointsInSpec) @@ -274,7 +290,7 @@ class ApiCoverageReportTest { Endpoint("/order/{id}", "GET", 200), Endpoint("/order/{id}", "GET", 400) ) val testResultRecords = mutableListOf( - TestResultRecord("/order/{id}", "GET", 200, TestResult.Success, actualResponseStatus = 200) + TestResultRecord("/order/{id}", "GET", 200, request = null, response = null, result = TestResult.Success, actualResponseStatus = 200) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec, filteredEndpoints = endpointsInSpec) @@ -296,8 +312,8 @@ class ApiCoverageReportTest { Endpoint("/order/{id}", "POST", 404) ) val testResultRecords = mutableListOf( - TestResultRecord("/order/{id}", "POST", 201, TestResult.Success, actualResponseStatus = 201), - TestResultRecord("/order/{id}", "POST", 400, TestResult.Success, actualResponseStatus = 400) + TestResultRecord("/order/{id}", "POST", 201, request = null, response = null, result = TestResult.Success, actualResponseStatus = 201), + TestResultRecord("/order/{id}", "POST", 400, request = null, response = null, result = TestResult.Success, actualResponseStatus = 400) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec, filteredEndpoints = endpointsInSpec) @@ -323,7 +339,7 @@ class ApiCoverageReportTest { API("GET", "/order/{id}") ) val testResultRecords = mutableListOf( - TestResultRecord("/order/{id}", "GET", 200, TestResult.Failed, actualResponseStatus = 404) + TestResultRecord("/order/{id}", "GET", 200, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec, applicationAPIs) @@ -343,7 +359,7 @@ class ApiCoverageReportTest { Endpoint("/order/{id}", "GET", 200) ) val testResultRecords = mutableListOf( - TestResultRecord("/order/{id}", "GET", 200, TestResult.Failed, actualResponseStatus = 404) + TestResultRecord("/order/{id}", "GET", 200, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec) @@ -366,7 +382,7 @@ class ApiCoverageReportTest { API("GET", "/order/{id}") ) val testResultRecords = mutableListOf( - TestResultRecord("/order/{id}", "GET", 200, TestResult.Failed, actualResponseStatus = 404) + TestResultRecord("/order/{id}", "GET", 200, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec, applicationAPIs, filteredEndpoints = endpointsInSpec) @@ -386,7 +402,7 @@ class ApiCoverageReportTest { Endpoint("/order/{id}", "GET", 200), Endpoint("/order/{id}", "GET", 404) ) val testResultRecords = mutableListOf( - TestResultRecord("/order/{id}", "GET", 200, TestResult.Failed, actualResponseStatus = 404) + TestResultRecord("/order/{id}", "GET", 200, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec, filteredEndpoints = endpointsInSpec) @@ -410,7 +426,7 @@ class ApiCoverageReportTest { val applicationAPIs = mutableListOf() val testResultRecords = mutableListOf( - TestResultRecord("/orders", "GET", 200, TestResult.Failed, actualResponseStatus = 404) + TestResultRecord("/orders", "GET", 200, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec, applicationAPIs) @@ -431,7 +447,7 @@ class ApiCoverageReportTest { ) val testResultRecords = mutableListOf( - TestResultRecord("/orders", "GET", 200, TestResult.Failed, actualResponseStatus = 404) + TestResultRecord("/orders", "GET", 200, request = null, response = null, result = TestResult.Failed, actualResponseStatus = 404) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec, filteredEndpoints = endpointsInSpec) @@ -457,7 +473,7 @@ class ApiCoverageReportTest { ) val testResultRecords = mutableListOf( - TestResultRecord("/orders", "GET", 200, TestResult.Success, actualResponseStatus = 200) + TestResultRecord("/orders", "GET", 200, request = null, response = null, result = TestResult.Success, actualResponseStatus = 200) ) val apiCoverageReport = generateCoverageReport(testResultRecords, endpointsInSpec, applicationAPIs, filteredEndpoints = endpointsInSpec) diff --git a/junit5-support/src/test/kotlin/io/specmatic/test/reports/coverage/OpenApiCoverageReportInputTest.kt b/junit5-support/src/test/kotlin/io/specmatic/test/reports/coverage/OpenApiCoverageReportInputTest.kt index 73838e40dc..bb30f5baf0 100644 --- a/junit5-support/src/test/kotlin/io/specmatic/test/reports/coverage/OpenApiCoverageReportInputTest.kt +++ b/junit5-support/src/test/kotlin/io/specmatic/test/reports/coverage/OpenApiCoverageReportInputTest.kt @@ -14,6 +14,8 @@ class OpenApiCoverageReportInputTest { path = "/current", method = "GET", responseStatus = 200, + request = null, + response = null, result = TestResult.Success ) @@ -34,6 +36,8 @@ class OpenApiCoverageReportInputTest { path = "/current", method = "GET", responseStatus = 200, + request = null, + response = null, result = TestResult.Success ) @@ -41,6 +45,8 @@ class OpenApiCoverageReportInputTest { path = "/previous", method = "POST", responseStatus = 201, + request = null, + response = null, result = TestResult.Success ) @@ -66,6 +72,8 @@ class OpenApiCoverageReportInputTest { path = "/current", method = "GET", responseStatus = 200, + request = null, + response = null, result = TestResult.Success ) @@ -95,6 +103,8 @@ class OpenApiCoverageReportInputTest { path = "/current", method = "GET", responseStatus = 200, + request = null, + response = null, result = TestResult.Success ) @@ -102,6 +112,8 @@ class OpenApiCoverageReportInputTest { path = "/previous", method = "POST", responseStatus = 201, + request = null, + response = null, result = TestResult.Success ) @@ -131,6 +143,8 @@ class OpenApiCoverageReportInputTest { path = "/resource", method = "GET", responseStatus = 200, + request = null, + response = null, result = TestResult.Success ) @@ -166,6 +180,8 @@ class OpenApiCoverageReportInputTest { path = "/resource", method = "GET", responseStatus = 200, + request = null, + response = null, result = TestResult.Success ) @@ -173,6 +189,8 @@ class OpenApiCoverageReportInputTest { path = "/resource", method = "POST", responseStatus = 201, + request = null, + response = null, result = TestResult.Success ) @@ -202,7 +220,7 @@ class OpenApiCoverageReportInputTest { val allEndpoints = mutableListOf(Endpoint("/test", "POST", 200), Endpoint("/filtered", "POST", 200)) val filtered = mutableListOf(Endpoint("/test", "POST", 200)) val testResultRecords = mutableListOf( - TestResultRecord("/test", "POST", 200, TestResult.Failed, actualResponseStatus = 200), + TestResultRecord("/test", "POST", 200,request = null, response = null, result = TestResult.Failed, actualResponseStatus = 200), ) val reportInput = OpenApiCoverageReportInput( @@ -222,7 +240,7 @@ class OpenApiCoverageReportInputTest { val filtered = mutableListOf(Endpoint("/test", "POST", 200)) val applicationAPIs = mutableListOf(API("POST", "/test"), API("POST", "/filtered")) val testResultRecords = mutableListOf( - TestResultRecord("/test", "POST", 200, TestResult.Failed, actualResponseStatus = 200), + TestResultRecord("/test", "POST", 200,request = null, response = null, result = TestResult.Failed, actualResponseStatus = 200), ) val reportInput = OpenApiCoverageReportInput( @@ -239,7 +257,7 @@ class OpenApiCoverageReportInputTest { fun `not-implemented endpoints should be identified using filtered endpoints instead of all endpoints`() { val allEndpoints = mutableListOf(Endpoint("/test", "POST", 200), Endpoint("/filtered", "POST", 200)) val filtered = mutableListOf(Endpoint("/test", "POST", 200)) - val testResultRecords = mutableListOf(TestResultRecord("/test", "POST", 200, TestResult.Failed, actualResponseStatus = 0)) + val testResultRecords = mutableListOf(TestResultRecord("/test", "POST", 200,request = null, response = null, result = TestResult.Failed, actualResponseStatus = 0)) val reportInput = OpenApiCoverageReportInput( testResultRecords = testResultRecords, configFilePath = "", endpointsAPISet = true, From b1b422e901f15a5e7e6ba58a8214945bb610a17d Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Mon, 1 Dec 2025 17:42:04 +0530 Subject: [PATCH 08/35] chore: fix tests --- .../conversions/OpenApiIntegrationTest.kt | 32 +++++++++---------- .../io/specmatic/conversions/OpenApiKtTest.kt | 4 +-- .../conversions/OpenApiSpecificationTest.kt | 8 ++--- .../kotlin/io/specmatic/core/FeatureTest.kt | 6 ++-- .../core/RunContractTestsUsingScenario.kt | 2 +- .../core/filters/TestRecordFilterTest.kt | 2 ++ .../io/specmatic/test/TestResultRecordTest.kt | 8 +++++ 7 files changed, 36 insertions(+), 26 deletions(-) diff --git a/core/src/test/kotlin/io/specmatic/conversions/OpenApiIntegrationTest.kt b/core/src/test/kotlin/io/specmatic/conversions/OpenApiIntegrationTest.kt index c50e72801e..f8ec3b2214 100644 --- a/core/src/test/kotlin/io/specmatic/conversions/OpenApiIntegrationTest.kt +++ b/core/src/test/kotlin/io/specmatic/conversions/OpenApiIntegrationTest.kt @@ -57,7 +57,7 @@ Examples: } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -78,7 +78,7 @@ Examples: } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -100,7 +100,7 @@ Examples: override fun setServerState(serverState: Map) { } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -124,7 +124,7 @@ Examples: override fun setServerState(serverState: Map) { } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) } finally { @@ -238,7 +238,7 @@ Feature: Authenticated } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -290,7 +290,7 @@ Feature: Authenticated } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -342,7 +342,7 @@ Feature: Authenticated } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -395,7 +395,7 @@ Feature: Authenticated } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -419,7 +419,7 @@ Feature: Authenticated override fun setServerState(serverState: Map) { } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) } assertThat(requestMadeWithRandomlyGeneratedBearerToken).isTrue @@ -448,7 +448,7 @@ Feature: Authenticated override fun setServerState(serverState: Map) { } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) } assertThat(requestMadeWithTokenFromSpecmaticJson).isTrue @@ -485,7 +485,7 @@ Feature: Authenticated override fun setServerState(serverState: Map) { } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) } assertThat(requestMadeWithTokenFromSpecmaticJson).isTrue @@ -745,7 +745,7 @@ Feature: Authenticated } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -808,7 +808,7 @@ Feature: Authenticated } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -836,7 +836,7 @@ Feature: Authenticated override fun setServerState(serverState: Map) { } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) } assertThat(requestMadeWithApiKeyInHeaderFromSpecmaticJson).isTrue @@ -871,7 +871,7 @@ Feature: Authenticated override fun setServerState(serverState: Map) { } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) } assertThat(requestMadeWithApiKeyInHeaderFromSpecmaticJson).isTrue @@ -905,7 +905,7 @@ Feature: Authenticated override fun setServerState(serverState: Map) { } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) } assertThat(requestMadeWithApiKeyInQueryFromSpecmaticJson).isTrue diff --git a/core/src/test/kotlin/io/specmatic/conversions/OpenApiKtTest.kt b/core/src/test/kotlin/io/specmatic/conversions/OpenApiKtTest.kt index 92a990c332..de011c3c7d 100644 --- a/core/src/test/kotlin/io/specmatic/conversions/OpenApiKtTest.kt +++ b/core/src/test/kotlin/io/specmatic/conversions/OpenApiKtTest.kt @@ -756,7 +756,7 @@ Feature: multipart file upload } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -1519,7 +1519,7 @@ Scenario: zero should return not found override fun setServerState(serverState: Map) { } - }).first + }).result assertThat(result).isInstanceOf(Result.Success::class.java) assertThat(executed).isTrue diff --git a/core/src/test/kotlin/io/specmatic/conversions/OpenApiSpecificationTest.kt b/core/src/test/kotlin/io/specmatic/conversions/OpenApiSpecificationTest.kt index b21a3d60da..cb49f97f7b 100644 --- a/core/src/test/kotlin/io/specmatic/conversions/OpenApiSpecificationTest.kt +++ b/core/src/test/kotlin/io/specmatic/conversions/OpenApiSpecificationTest.kt @@ -5540,7 +5540,7 @@ paths: override fun setServerState(serverState: Map) { } - }).first + }).result } assertThat(results).hasSize(1) @@ -5614,7 +5614,7 @@ paths: override fun setServerState(serverState: Map) { } - }).first + }).result } assertThat(results).hasSize(1) @@ -5676,7 +5676,7 @@ paths: override fun setServerState(serverState: Map) { } - }).first + }).result } assertThat(results).hasSize(1) @@ -5738,7 +5738,7 @@ paths: override fun setServerState(serverState: Map) { } - }).first + }).result } assertThat(results).hasSize(1) diff --git a/core/src/test/kotlin/io/specmatic/core/FeatureTest.kt b/core/src/test/kotlin/io/specmatic/core/FeatureTest.kt index 0ed55d8371..94a96f9d17 100644 --- a/core/src/test/kotlin/io/specmatic/core/FeatureTest.kt +++ b/core/src/test/kotlin/io/specmatic/core/FeatureTest.kt @@ -2601,7 +2601,7 @@ paths: println(it.toLogString()) } } - }).first + }).result assertThat(results.isSuccess()).isTrue() } @@ -2702,7 +2702,7 @@ paths: println(it.toLogString()) } } - }).first + }).result assertThat(result.isSuccess()).withFailMessage(result.reportString()).isTrue() } @@ -2806,7 +2806,7 @@ paths: println(it.toLogString()) } } - }).first + }).result assertThat(result).isInstanceOf(Result.Failure::class.java) assertThat(result.reportString()).isEqualToNormalizingWhitespace(""" diff --git a/core/src/test/kotlin/io/specmatic/core/RunContractTestsUsingScenario.kt b/core/src/test/kotlin/io/specmatic/core/RunContractTestsUsingScenario.kt index 895175dc7b..2cfc5af5ec 100644 --- a/core/src/test/kotlin/io/specmatic/core/RunContractTestsUsingScenario.kt +++ b/core/src/test/kotlin/io/specmatic/core/RunContractTestsUsingScenario.kt @@ -529,7 +529,7 @@ paths: override fun setServerState(serverState: Map) { } - }).first as Result.Failure + }).result as Result.Failure assertThat(result.reportString()).contains("Contract expected") assertThat(result.reportString()).contains("response contained") diff --git a/core/src/test/kotlin/io/specmatic/core/filters/TestRecordFilterTest.kt b/core/src/test/kotlin/io/specmatic/core/filters/TestRecordFilterTest.kt index 24f90c0a22..610a814ea3 100644 --- a/core/src/test/kotlin/io/specmatic/core/filters/TestRecordFilterTest.kt +++ b/core/src/test/kotlin/io/specmatic/core/filters/TestRecordFilterTest.kt @@ -12,6 +12,8 @@ class TestRecordFilterTest { path = path, method = method, responseStatus = responseStatus, + request = null, + response = null, result = TestResult.Success ) diff --git a/core/src/test/kotlin/io/specmatic/test/TestResultRecordTest.kt b/core/src/test/kotlin/io/specmatic/test/TestResultRecordTest.kt index 9bc3890250..807326085f 100644 --- a/core/src/test/kotlin/io/specmatic/test/TestResultRecordTest.kt +++ b/core/src/test/kotlin/io/specmatic/test/TestResultRecordTest.kt @@ -14,6 +14,8 @@ class TestResultRecordTest { path = "/example/path", method = "GET", responseStatus = 200, + request = null, + response = null, result = it ) assertFalse(record.isExercised, "Record should not be considered exercised for Result: $it") @@ -27,6 +29,8 @@ class TestResultRecordTest { path = "/example/path", method = "GET", responseStatus = 200, + request = null, + response = null, result = it ) assertTrue(record.isExercised, "Record should be considered exercised for Result: $it") @@ -40,6 +44,8 @@ class TestResultRecordTest { path = "/example/path", method = "GET", responseStatus = 200, + request = null, + response = null, result = it ) assertTrue(record.isCovered, "Record should be considered covered for result $it") @@ -53,6 +59,8 @@ class TestResultRecordTest { path = "/example/path", method = "GET", responseStatus = 200, + request = null, + response = null, result = it ) assertFalse(record.isCovered, "Record should not be considered covered for result $it") From 98368c469cf31b70e1527464176d3edeefabf4cc Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Tue, 2 Dec 2025 12:26:23 +0530 Subject: [PATCH 09/35] chore: remove serverUrl from CtrfSpecConfig --- .../src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt b/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt index a52ac963c5..1d44d70d1d 100644 --- a/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt +++ b/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt @@ -204,8 +204,7 @@ open class SpecmaticJUnitSupport { specification = it.specification.orEmpty(), sourceProvider = it.sourceProvider, repository = it.sourceRepository, - branch = it.sourceRepositoryBranch, - serverUrl = "" + branch = it.sourceRepositoryBranch ) } } From 6c4c96e676c4749fc17564818dcbbfd50e3cfb92 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Wed, 3 Dec 2025 08:50:21 +0530 Subject: [PATCH 10/35] chore: update specmatic-reporter version to 0.1.8 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index fb5102d352..5d713b294c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ group=io.specmatic version=2.32.2-SNAPSHOT specmaticGradlePluginVersion=0.13.8 -specmaticReporterVersion=0.1.7 +specmaticReporterVersion=0.1.8 kotlin.daemon.jvmargs=-Xmx1024m org.gradle.jvmargs=-Xmx1024m \ No newline at end of file From 2da47d6d710ffa440d7cad86e068c1fbe548d7fe Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Wed, 3 Dec 2025 11:34:29 +0530 Subject: [PATCH 11/35] chore: add test for extraFields method in TestResultRecord --- .../io/specmatic/test/TestResultRecordTest.kt | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/core/src/test/kotlin/io/specmatic/test/TestResultRecordTest.kt b/core/src/test/kotlin/io/specmatic/test/TestResultRecordTest.kt index 807326085f..afb5fd23f0 100644 --- a/core/src/test/kotlin/io/specmatic/test/TestResultRecordTest.kt +++ b/core/src/test/kotlin/io/specmatic/test/TestResultRecordTest.kt @@ -1,9 +1,13 @@ package io.specmatic.test +import io.specmatic.core.HttpRequest +import io.specmatic.core.HttpResponse import io.specmatic.reporter.model.TestResult +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import java.time.Instant class TestResultRecordTest { @@ -66,4 +70,64 @@ class TestResultRecordTest { assertFalse(record.isCovered, "Record should not be considered covered for result $it") } } + + @Test + fun `extraFields should reflect request and response and times when present`() { + val request = HttpRequest( + method = "POST", + path = "/some/path", + headers = mapOf("Content-Type" to "application/json") + ) + val response = HttpResponse.ok("{\"hello\":\"world\"}") + + val requestTime = Instant.ofEpochMilli(1_000L) + val responseTime = Instant.ofEpochMilli(2_000L) + + val record = TestResultRecord( + path = "/some/path", + method = "POST", + responseStatus = 200, + request = request, + response = response, + result = TestResult.Success, + isValid = true, + isWip = false, + requestTime = requestTime, + responseTime = responseTime + ) + + val meta = record.extraFields() + + assertTrue(meta.valid) + assertFalse(meta.isWip) + assertEquals(request.toLogString().trim(), meta.input.trim()) + assertEquals(response.toLogString().trim(), meta.output?.trim()) + assertEquals(requestTime.toEpochMilli(), meta.inputTime) + assertEquals(responseTime.toEpochMilli(), meta.outputTime) + } + + @Test + fun `extraFields should use defaults when request or response are null`() { + val record = TestResultRecord( + path = "/some/path", + method = "GET", + responseStatus = 200, + request = null, + response = null, + result = TestResult.Success, + isValid = false, + isWip = true, + requestTime = null, + responseTime = null + ) + + val meta = record.extraFields() + + assertFalse(meta.valid) + assertTrue(meta.isWip) + assertEquals("", meta.input) + assertEquals("", meta.output) + assertEquals(0L, meta.inputTime) + assertEquals(0L, meta.outputTime) + } } From 4c80e961e9914993debddd64bedf1abb1b7c7765 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Wed, 3 Dec 2025 09:51:33 +0530 Subject: [PATCH 12/35] Ensure public APIs can load WSDL externalized examples - Add simple tests for the same --- core/src/main/kotlin/io/specmatic/stub/api.kt | 6 ++- .../kotlin/io/specmatic/stub/ApiKtTest.kt | 53 +++++++++++++++++++ .../order_api_examples/create_product.json | 1 + 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/core/src/main/kotlin/io/specmatic/stub/api.kt b/core/src/main/kotlin/io/specmatic/stub/api.kt index 1c1687534d..e449ba4ee0 100644 --- a/core/src/main/kotlin/io/specmatic/stub/api.kt +++ b/core/src/main/kotlin/io/specmatic/stub/api.kt @@ -484,7 +484,7 @@ fun loadContractStubsFromFilesAsResults( specmaticConfig = specmaticConfig, externalDataDirPaths = dataDirPaths, cachedFeatures = features.map { it.second }, - processedInvalidSpecs = contractPathDataList.filter { isInvalidOpenAPISpecification(it.path) }.map { it.path }, + processedInvalidSpecs = contractPathDataList.excludingSpecifications().map { it.path }, ) return explicitStubs.plus(implicitStubs) @@ -1063,7 +1063,9 @@ fun loadIfSupportedAPISpecification( } } -fun isInvalidOpenAPISpecification(specPath: String): Boolean = hasOpenApiFileExtension(specPath).not() || isOpenAPI(specPath).not() +private fun List.excludingSpecifications(): List { + return this.filterNot { isSupportedAPISpecification(it.path) } +} fun isOpenAPI(path: String): Boolean = try { diff --git a/core/src/test/kotlin/io/specmatic/stub/ApiKtTest.kt b/core/src/test/kotlin/io/specmatic/stub/ApiKtTest.kt index 5dcc8c0534..605bb5b9ea 100644 --- a/core/src/test/kotlin/io/specmatic/stub/ApiKtTest.kt +++ b/core/src/test/kotlin/io/specmatic/stub/ApiKtTest.kt @@ -648,6 +648,59 @@ Feature: Math API assertThat(output).contains("Skipping the file") assertThat(output).endsWith("as it is not a supported API specification") } + + @Test + fun `loadIfSupportedAPISpecification should be able to load WSDL specifications`() { + val specFile = File("src/test/resources/wsdl/hello.wsdl") + val contractPathData = ContractPathData("", specFile.path) + val result = loadIfSupportedAPISpecification(contractPathData, SpecmaticConfig()) + + assertThat(result).isNotNull + assertThat(result!!.first).isEqualTo(specFile.path) + assertThat(result.second.scenarios.first().isGherkinScenario).isTrue + } + + @Test + fun `loadContractStubsFromFilesAsResults should be able to load WSDL specifications with external examples`() { + val specFile = File("src/test/resources/wsdl/with_examples/order_api.wsdl") + val contractPathData = listOf(ContractPathData("", specFile.path)) + val result = loadContractStubsFromFilesAsResults(contractPathData, emptyList(), SpecmaticConfig(), withImplicitStubs = true) + + assertThat(result).allSatisfy { + assertThat(it).isInstanceOf(FeatureStubsResult.Success::class.java); it as FeatureStubsResult.Success + assertThat(it.feature.scenarios.first().isGherkinScenario).isTrue + } + + assertThat(result).anySatisfy { + assertThat(it).isInstanceOf(FeatureStubsResult.Success::class.java); it as FeatureStubsResult.Success + assertThat(it.scenarioStubs).hasSize(1) + assertThat(it.scenarioStubs.single().name).isEqualTo("createProduct") + } + } + + @Test + fun `loadContractStubsFromFilesAsResults should return failure to load WSDL specifications with invalid external examples`(@TempDir tempDir: File) { + File("src/test/resources/wsdl/with_examples").copyRecursively(tempDir) + val specFile = tempDir.resolve("order_api.wsdl") + val example = tempDir.resolve("order_api_examples/create_product.json") + val contractPathData = listOf(ContractPathData("", specFile.path)) + + example.writeText(example.readText().replace("orders/createProduct", "orders/createProductDoesNotExist")) + val result = loadContractStubsFromFilesAsResults(contractPathData, emptyList(), SpecmaticConfig(), withImplicitStubs = true) + + assertThat(result).anySatisfy { + assertThat(it).isInstanceOf(FeatureStubsResult.Success::class.java); it as FeatureStubsResult.Success + assertThat(it.feature.scenarios.first().isGherkinScenario).isTrue + } + + assertThat(result).anySatisfy { + assertThat(it).isInstanceOf(FeatureStubsResult.Failure::class.java); it as FeatureStubsResult.Failure + assertThat(it.stubFile).endsWith("create_product.json") + assertThat(it.errorMessage).containsIgnoringWhitespaces(""" + No matching SOAP stub or contract found for SOAPAction "/orders/createProductDoesNotExist" and path / + """.trimIndent()) + } + } } fun captureStandardOutput(trim: Boolean = true, fn: () -> ReturnType): Pair { diff --git a/core/src/test/resources/wsdl/with_examples/order_api_examples/create_product.json b/core/src/test/resources/wsdl/with_examples/order_api_examples/create_product.json index c50136fd37..2f44d454b7 100644 --- a/core/src/test/resources/wsdl/with_examples/order_api_examples/create_product.json +++ b/core/src/test/resources/wsdl/with_examples/order_api_examples/create_product.json @@ -1,4 +1,5 @@ { + "name": "createProduct", "http-request": { "method": "POST", "path": "/", From 63e5668f7f67701aeb2c2734e1711d51d0e503a0 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Wed, 3 Dec 2025 10:26:30 +0530 Subject: [PATCH 13/35] Resolve the Windows CI build failure - Caused due to attempts made to relativize paths with different roots - These paths are only used for logging so should be harmless --- core/src/main/kotlin/io/specmatic/stub/api.kt | 18 +++-- .../kotlin/io/specmatic/stub/ApiKtTest.kt | 8 +-- .../wsdl/with_invalid_examples/order_api.wsdl | 72 +++++++++++++++++++ .../order_api_examples/create_product.json | 19 +++++ 4 files changed, 106 insertions(+), 11 deletions(-) create mode 100644 core/src/test/resources/wsdl/with_invalid_examples/order_api.wsdl create mode 100644 core/src/test/resources/wsdl/with_invalid_examples/order_api_examples/create_product.json diff --git a/core/src/main/kotlin/io/specmatic/stub/api.kt b/core/src/main/kotlin/io/specmatic/stub/api.kt index e449ba4ee0..b6ce797745 100644 --- a/core/src/main/kotlin/io/specmatic/stub/api.kt +++ b/core/src/main/kotlin/io/specmatic/stub/api.kt @@ -566,11 +566,19 @@ private fun List.withAbsolutePaths(): String = ) } -private fun List.relativePaths(): List = - this - .map { - File(it).canonicalFile.relativeTo(File(".").canonicalFile).path - }.map { ".${File.separator}$it" } +private fun List.relativePaths(): List { + val currentWorkingDirectory = File(".").canonicalFile + return this.map(::File).map { + runCatching { + it.canonicalFile.relativeTo(currentWorkingDirectory) + }.getOrElse { e -> + logger.debug(e, "Failed to relativize ${it.canonicalPath} against ${currentWorkingDirectory.canonicalPath}") + it + } + }.map { + ".${File.separator}${it.path}" + } +} private fun List>.overrideInlineExamplesWithSameNameFrom(dataFiles: List): List> { val externalExampleNames = dataFiles.map { it.nameWithoutExtension }.toSet() diff --git a/core/src/test/kotlin/io/specmatic/stub/ApiKtTest.kt b/core/src/test/kotlin/io/specmatic/stub/ApiKtTest.kt index 605bb5b9ea..a278f9fbfb 100644 --- a/core/src/test/kotlin/io/specmatic/stub/ApiKtTest.kt +++ b/core/src/test/kotlin/io/specmatic/stub/ApiKtTest.kt @@ -679,13 +679,9 @@ Feature: Math API } @Test - fun `loadContractStubsFromFilesAsResults should return failure to load WSDL specifications with invalid external examples`(@TempDir tempDir: File) { - File("src/test/resources/wsdl/with_examples").copyRecursively(tempDir) - val specFile = tempDir.resolve("order_api.wsdl") - val example = tempDir.resolve("order_api_examples/create_product.json") + fun `loadContractStubsFromFilesAsResults should return failure to load WSDL specifications with invalid external examples`() { + val specFile = File("src/test/resources/wsdl/with_invalid_examples/order_api.wsdl") val contractPathData = listOf(ContractPathData("", specFile.path)) - - example.writeText(example.readText().replace("orders/createProduct", "orders/createProductDoesNotExist")) val result = loadContractStubsFromFilesAsResults(contractPathData, emptyList(), SpecmaticConfig(), withImplicitStubs = true) assertThat(result).anySatisfy { diff --git a/core/src/test/resources/wsdl/with_invalid_examples/order_api.wsdl b/core/src/test/resources/wsdl/with_invalid_examples/order_api.wsdl new file mode 100644 index 0000000000..7be132635f --- /dev/null +++ b/core/src/test/resources/wsdl/with_invalid_examples/order_api.wsdl @@ -0,0 +1,72 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/core/src/test/resources/wsdl/with_invalid_examples/order_api_examples/create_product.json b/core/src/test/resources/wsdl/with_invalid_examples/order_api_examples/create_product.json new file mode 100644 index 0000000000..2d8468f7fc --- /dev/null +++ b/core/src/test/resources/wsdl/with_invalid_examples/order_api_examples/create_product.json @@ -0,0 +1,19 @@ +{ + "name": "createProductDoesNotExist", + "http-request": { + "method": "POST", + "path": "/", + "headers": { + "SOAPAction": "\"/orders/createProductDoesNotExist\"", + "Content-Type": "text/xml" + }, + "body": "PhoneGadget" + }, + "http-response": { + "status": 200, + "headers": { + "Content-Type": "text/xml" + }, + "body": "123" + } +} From 7eee6587ec390f47ec60ed7c6988c1fa5864bc69 Mon Sep 17 00:00:00 2001 From: Joel Rosario Date: Wed, 3 Dec 2025 15:03:01 +0530 Subject: [PATCH 14/35] Renamed function for clarity --- core/src/main/kotlin/io/specmatic/stub/api.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/kotlin/io/specmatic/stub/api.kt b/core/src/main/kotlin/io/specmatic/stub/api.kt index b6ce797745..59913e969f 100644 --- a/core/src/main/kotlin/io/specmatic/stub/api.kt +++ b/core/src/main/kotlin/io/specmatic/stub/api.kt @@ -484,7 +484,7 @@ fun loadContractStubsFromFilesAsResults( specmaticConfig = specmaticConfig, externalDataDirPaths = dataDirPaths, cachedFeatures = features.map { it.second }, - processedInvalidSpecs = contractPathDataList.excludingSpecifications().map { it.path }, + processedInvalidSpecs = contractPathDataList.excludeUnsupportedSpecifications().map { it.path }, ) return explicitStubs.plus(implicitStubs) @@ -1071,7 +1071,7 @@ fun loadIfSupportedAPISpecification( } } -private fun List.excludingSpecifications(): List { +private fun List.excludeUnsupportedSpecifications(): List { return this.filterNot { isSupportedAPISpecification(it.path) } } From d127fa8396688e12281cd273b53992389dd68702 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Wed, 3 Dec 2025 17:12:04 +0530 Subject: [PATCH 15/35] feat: implement ctrf report generation for stub --- .../core/report/CtrfSpecConfigUtils.kt | 66 +++++++++++++++++++ .../core/report}/SpecmaticAfterAllHook.kt | 9 +-- .../main/kotlin/io/specmatic/stub/HttpStub.kt | 40 +++++++++++ .../io/specmatic/test/TestResultRecord.kt | 5 +- .../specmatic/test/SpecmaticJUnitSupport.kt | 8 ++- 5 files changed, 120 insertions(+), 8 deletions(-) create mode 100644 core/src/main/kotlin/io/specmatic/core/report/CtrfSpecConfigUtils.kt rename {junit5-support/src/main/kotlin/io/specmatic/test => core/src/main/kotlin/io/specmatic/core/report}/SpecmaticAfterAllHook.kt (53%) diff --git a/core/src/main/kotlin/io/specmatic/core/report/CtrfSpecConfigUtils.kt b/core/src/main/kotlin/io/specmatic/core/report/CtrfSpecConfigUtils.kt new file mode 100644 index 0000000000..8da0f18de0 --- /dev/null +++ b/core/src/main/kotlin/io/specmatic/core/report/CtrfSpecConfigUtils.kt @@ -0,0 +1,66 @@ +package io.specmatic.core.report + +import io.specmatic.core.Source +import io.specmatic.core.SpecmaticConfig +import io.specmatic.core.config.v3.SpecExecutionConfig +import io.specmatic.reporter.ctrf.model.CtrfSpecConfig +import io.specmatic.reporter.ctrf.model.CtrfTestResultRecord +import io.specmatic.test.TestResultRecord.Companion.CONTRACT_TEST_TEST_TYPE +import java.io.File + +fun ctrfSpecConfigsFrom( + specmaticConfig: SpecmaticConfig, + testResultRecords: List, + serviceType: String = "HTTP", + specType: String = "OPENAPI", +): List { + val specConfigs = testResultRecords.map { + it.specification.orEmpty() to it.testType + }.map { (specification, testType) -> + val source = associatedSource(specification, specmaticConfig, testType) + CtrfSpecConfig( + serviceType = serviceType, + specType = specType, + specification = specification, + sourceProvider = source.provider.name, + repository = source.repository.orEmpty(), + branch = source.branch ?: "main", + ) + } + return specConfigs +} + +private fun associatedSource(specification: String, specmaticConfig: SpecmaticConfig, testType: String): Source { + val specName = File(specification).name + val source = when (testType) { + CONTRACT_TEST_TEST_TYPE -> testSourceFromConfig(specName, specmaticConfig) + else -> stubSourceFromConfig(specName, specmaticConfig) + } + return source ?: Source() +} + +private fun testSourceFromConfig(specificationName: String, specmaticConfig: SpecmaticConfig): Source? { + val sources = SpecmaticConfig.getSources(specmaticConfig) + return sources.firstOrNull { source -> + source.test.orEmpty().any { test -> + test.contains(specificationName) + } + } +} + + +private fun stubSourceFromConfig(specificationName: String, specmaticConfig: SpecmaticConfig): Source? { + val sources = SpecmaticConfig.getSources(specmaticConfig) + return sources.firstOrNull { source -> + source.stub.orEmpty().any { stub -> + when (stub) { + is SpecExecutionConfig.StringValue -> stub.value.contains(specificationName) + is SpecExecutionConfig.ObjectValue -> stub.specs.any { it.contains(specificationName) } + } + } + } +} + + + + diff --git a/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticAfterAllHook.kt b/core/src/main/kotlin/io/specmatic/core/report/SpecmaticAfterAllHook.kt similarity index 53% rename from junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticAfterAllHook.kt rename to core/src/main/kotlin/io/specmatic/core/report/SpecmaticAfterAllHook.kt index a90cd9d1d4..4a57f29895 100644 --- a/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticAfterAllHook.kt +++ b/core/src/main/kotlin/io/specmatic/core/report/SpecmaticAfterAllHook.kt @@ -1,13 +1,14 @@ -package io.specmatic.test +package io.specmatic.core.report import io.specmatic.reporter.ctrf.model.CtrfSpecConfig +import io.specmatic.test.TestResultRecord interface SpecmaticAfterAllHook { - fun onAfterAllTests( + fun generateReport( testResultRecords: List?, startTime: Long, endTime: Long, - coverage: Int, - specConfigs: List + specConfigs: List, + reportFilePath: String ) } \ No newline at end of file diff --git a/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt b/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt index ee95d3aa01..dc2772f1a3 100644 --- a/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt +++ b/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt @@ -14,6 +14,7 @@ import io.ktor.util.* import io.ktor.util.pipeline.* import io.specmatic.core.APPLICATION_NAME import io.specmatic.core.APPLICATION_NAME_LOWER_CASE +import io.specmatic.core.Constants.Companion.ARTIFACTS_PATH import io.specmatic.core.ContractAndStubMismatchMessages import io.specmatic.core.Feature import io.specmatic.core.HttpRequest @@ -35,6 +36,7 @@ import io.specmatic.core.Scenario import io.specmatic.core.SpecmaticConfig import io.specmatic.core.WorkingDirectory import io.specmatic.core.listOfExcludedHeaders +import io.specmatic.core.loadSpecmaticConfigOrDefault import io.specmatic.core.log.HttpLogMessage import io.specmatic.core.log.LogMessage import io.specmatic.core.log.LogTail @@ -47,6 +49,8 @@ import io.specmatic.core.parseGherkinStringToFeature import io.specmatic.core.pattern.ContractException import io.specmatic.core.pattern.parsedJSON import io.specmatic.core.pattern.parsedValue +import io.specmatic.core.report.SpecmaticAfterAllHook +import io.specmatic.core.report.ctrfSpecConfigsFrom import io.specmatic.core.route.modules.HealthCheckModule.Companion.configureHealthCheckModule import io.specmatic.core.route.modules.HealthCheckModule.Companion.isHealthCheckRequest import io.specmatic.core.urlDecodePathSegments @@ -71,11 +75,14 @@ import io.specmatic.mock.mockFromJSON import io.specmatic.mock.validateMock import io.specmatic.reporter.generated.dto.stub.usage.SpecmaticStubUsageReport import io.specmatic.reporter.internal.dto.stub.usage.merge +import io.specmatic.reporter.model.TestResult import io.specmatic.stub.listener.MockEvent import io.specmatic.stub.listener.MockEventListener import io.specmatic.stub.report.StubEndpoint import io.specmatic.stub.report.StubUsageReport import io.specmatic.test.LegacyHttpClient +import io.specmatic.test.TestResultRecord +import io.specmatic.test.TestResultRecord.Companion.STUB_TEST_TYPE import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.channels.BroadcastChannel import kotlinx.coroutines.channels.BufferOverflow @@ -91,6 +98,7 @@ import java.io.Writer import java.net.InetAddress import java.net.URI import java.nio.charset.Charset +import java.time.Instant import java.util.* import kotlin.text.toCharArray @@ -116,6 +124,7 @@ class HttpStub( }, private val listeners: List = emptyList(), private val reportBaseDirectoryPath: String = ".", + private val startTime: Instant = Instant.now() ) : ContractStub { constructor( feature: Feature, @@ -192,6 +201,8 @@ class HttpStub( private val specmaticConfigInstance: SpecmaticConfig = loadedSpecmaticConfig.config val specmaticConfigPath: String? = loadedSpecmaticConfig.path + private val ctrfTestResultRecords = mutableListOf() + val specToBaseUrlMap: Map = getValidatedBaseUrlsOrExit( features.associate { val baseUrl = specToStubBaseUrlMap[it.path] ?: endPointFromHostAndPort(host, port, keyData) @@ -352,6 +363,21 @@ class HttpStub( // Add the original response (before encoding) to the log message httpLogMessage.addResponse(httpStubResponse) } + + val ctrfTestResultRecord = TestResultRecord( + path = httpRequest.path, + method = httpRequest.method.orEmpty(), + responseStatus = httpResponse.status, + request = httpRequest, + response = httpResponse, + result = if(responseErrors.isEmpty()) TestResult.Success else TestResult.Failed, + serviceType = "OPENAPI", + requestContentType = httpRequest.headers["Content-Type"], + specification = httpStubResponse.scenario?.specification, + testType = STUB_TEST_TYPE, + actualResponseStatus = httpResponse.status + ) + synchronized(ctrfTestResultRecords) { ctrfTestResultRecords.add(ctrfTestResultRecord) } } catch (e: ContractException) { val response = badRequest(e.report()) httpLogMessage.addResponseWithCurrentTime(response) @@ -764,6 +790,20 @@ class HttpStub( override fun close() { server.stop(gracePeriodMillis = timeoutMillis, timeoutMillis = timeoutMillis) printUsageReport() + val specmaticConfig = loadSpecmaticConfigOrDefault(specmaticConfigPath) + synchronized(ctrfTestResultRecords) { + ServiceLoader.load(SpecmaticAfterAllHook::class.java).takeIf(ServiceLoader::any)?.let { hooks -> + hooks.forEach { + it.generateReport( + testResultRecords = ctrfTestResultRecords, + startTime = startTime.toEpochMilli(), + endTime = Instant.now().toEpochMilli(), + specConfigs = ctrfSpecConfigsFrom(specmaticConfig, ctrfTestResultRecords), + reportFilePath = "$ARTIFACTS_PATH/stub/ctrf/ctrf-report.json" + ) + } + } + } } private fun handleStateSetupRequest(httpRequest: HttpRequest): HttpStubResponse { diff --git a/core/src/main/kotlin/io/specmatic/test/TestResultRecord.kt b/core/src/main/kotlin/io/specmatic/test/TestResultRecord.kt index 3455116899..cceff8c711 100644 --- a/core/src/main/kotlin/io/specmatic/test/TestResultRecord.kt +++ b/core/src/main/kotlin/io/specmatic/test/TestResultRecord.kt @@ -36,7 +36,7 @@ data class TestResultRecord( val responseTime: Instant? = null, override val duration: Long = durationFrom(requestTime, responseTime), override val rawStatus: String? = result.toString(), - override val testType: String = "ContractTest", + override val testType: String = CONTRACT_TEST_TEST_TYPE, override val operation: APIOperation = OpenAPIOperation( path = path, method = method, @@ -98,6 +98,9 @@ data class TestResultRecord( } companion object { + const val STUB_TEST_TYPE = "Mock" + const val CONTRACT_TEST_TEST_TYPE = "ContractTest" + fun List.getCoverageStatus(): CoverageStatus { if(this.any { it.isWip }) return CoverageStatus.WIP diff --git a/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt b/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt index 1d44d70d1d..b71207310d 100644 --- a/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt +++ b/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt @@ -3,6 +3,7 @@ package io.specmatic.test import io.specmatic.conversions.OpenApiSpecification import io.specmatic.conversions.convertPathParameterStyle import io.specmatic.core.* +import io.specmatic.core.Constants.Companion.ARTIFACTS_PATH import io.specmatic.core.SpecmaticConfig.Companion.getSecurityConfiguration import io.specmatic.core.ResiliencyTestSuite import io.specmatic.core.filters.ScenarioMetadataFilter @@ -14,6 +15,7 @@ import io.specmatic.core.pattern.ContractException import io.specmatic.core.pattern.Examples import io.specmatic.core.pattern.Row import io.specmatic.core.pattern.parsedValue +import io.specmatic.core.report.SpecmaticAfterAllHook import io.specmatic.core.utilities.* import io.specmatic.core.utilities.Flags import io.specmatic.core.utilities.Flags.Companion.SPECMATIC_TEST_TIMEOUT @@ -209,12 +211,12 @@ open class SpecmaticJUnitSupport { } } hooks.forEach { - it.onAfterAllTests( + it.generateReport( testResultRecords = report.testResultRecords, - coverage = report.totalCoveragePercentage, startTime = start, endTime = end, - specConfigs = specConfigs + specConfigs = specConfigs, + reportFilePath = "$ARTIFACTS_PATH/test/ctrf/ctrf-report.json" ) } } From 260c1502f7c7d6459b650eed12c8a10786b24681 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Wed, 3 Dec 2025 17:21:31 +0530 Subject: [PATCH 16/35] fix: fix failing test --- application/src/test/kotlin/application/HTTPStubEngineTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/test/kotlin/application/HTTPStubEngineTest.kt b/application/src/test/kotlin/application/HTTPStubEngineTest.kt index 9b61657ace..08b2490d1b 100644 --- a/application/src/test/kotlin/application/HTTPStubEngineTest.kt +++ b/application/src/test/kotlin/application/HTTPStubEngineTest.kt @@ -68,7 +68,7 @@ class HTTPStubEngineTest { )) } - assertThat(stdOut).isEqualToNormalizingNewlines(""" + assertThat(stdOut).containsIgnoringNewLines(""" |Stub server is running on the following URLs: |- http://localhost:8000/api/v3 serving endpoints from specs: |\t1. api.yaml @@ -88,7 +88,7 @@ class HTTPStubEngineTest { )) } - assertThat(stdOut).isEqualToNormalizingNewlines(""" + assertThat(stdOut).containsIgnoringNewLines(""" |Stub server is running on the following URLs: |- http://0.0.0.0:9000 serving endpoints from specs: |\t1. api.yaml From bf092e0ce5951a894d53b1cbbfedb888811e202b Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Wed, 3 Dec 2025 18:06:17 +0530 Subject: [PATCH 17/35] refactor: refactor the ctrfspecconfig generation logic --- .../io/specmatic/core/SpecmaticConfig.kt | 51 +++++++++++++++++-- .../core/report/CtrfSpecConfigUtils.kt | 45 +--------------- 2 files changed, 47 insertions(+), 49 deletions(-) diff --git a/core/src/main/kotlin/io/specmatic/core/SpecmaticConfig.kt b/core/src/main/kotlin/io/specmatic/core/SpecmaticConfig.kt index 115f35186c..519a1931a1 100644 --- a/core/src/main/kotlin/io/specmatic/core/SpecmaticConfig.kt +++ b/core/src/main/kotlin/io/specmatic/core/SpecmaticConfig.kt @@ -51,7 +51,9 @@ import io.specmatic.core.utilities.exceptionCauseMessage import io.specmatic.core.utilities.readEnvVarOrProperty import io.specmatic.core.value.JSONObjectValue import io.specmatic.core.value.Value +import io.specmatic.reporter.ctrf.model.CtrfSpecConfig import io.specmatic.stub.isSameBaseIgnoringHost +import io.specmatic.test.TestResultRecord.Companion.CONTRACT_TEST_TEST_TYPE import java.io.File import java.net.URI @@ -303,6 +305,24 @@ data class SpecmaticConfig( } } + @JsonIgnore + fun getCtrfSpecConfig(specification: String, testType: String, serviceType: String, specType: String): CtrfSpecConfig { + val specName = File(specification).name + val source = when (testType) { + CONTRACT_TEST_TEST_TYPE -> testSourceFromConfig(specName) + else -> stubSourceFromConfig(specName) + } ?: Source() + + return CtrfSpecConfig( + serviceType = serviceType, + specType = specType, + specification = specification, + sourceProvider = source.provider.name, + repository = source.repository.orEmpty(), + branch = source.branch ?: "main", + ) + } + @JsonIgnore fun getHotReload(): Switch? { return stub.getHotReload() @@ -638,11 +658,6 @@ data class SpecmaticConfig( } } - @JsonIgnore - private fun String.canonicalPath(relativeTo: File): String { - return relativeTo.parentFile?.resolve(this)?.canonicalPath ?: File(this).canonicalPath - } - fun updateReportConfiguration(reportConfiguration: ReportConfiguration): SpecmaticConfig { val reportConfigurationDetails = reportConfiguration as? ReportConfigurationDetails ?: return this return this.copy(report = reportConfigurationDetails) @@ -669,6 +684,32 @@ data class SpecmaticConfig( ), ) } + + @JsonIgnore + private fun testSourceFromConfig(specificationName: String): Source? { + return sources.firstOrNull { source -> + source.test.orEmpty().any { test -> + test.contains(specificationName) + } + } + } + + @JsonIgnore + private fun stubSourceFromConfig(specificationName: String): Source? { + return sources.firstOrNull { source -> + source.stub.orEmpty().any { stub -> + when (stub) { + is SpecExecutionConfig.StringValue -> stub.value.contains(specificationName) + is SpecExecutionConfig.ObjectValue -> stub.specs.any { it.contains(specificationName) } + } + } + } + } + + @JsonIgnore + private fun String.canonicalPath(relativeTo: File): String { + return relativeTo.parentFile?.resolve(this)?.canonicalPath ?: File(this).canonicalPath + } } data class TestConfiguration( diff --git a/core/src/main/kotlin/io/specmatic/core/report/CtrfSpecConfigUtils.kt b/core/src/main/kotlin/io/specmatic/core/report/CtrfSpecConfigUtils.kt index 8da0f18de0..1b3e8bd06a 100644 --- a/core/src/main/kotlin/io/specmatic/core/report/CtrfSpecConfigUtils.kt +++ b/core/src/main/kotlin/io/specmatic/core/report/CtrfSpecConfigUtils.kt @@ -1,12 +1,8 @@ package io.specmatic.core.report -import io.specmatic.core.Source import io.specmatic.core.SpecmaticConfig -import io.specmatic.core.config.v3.SpecExecutionConfig import io.specmatic.reporter.ctrf.model.CtrfSpecConfig import io.specmatic.reporter.ctrf.model.CtrfTestResultRecord -import io.specmatic.test.TestResultRecord.Companion.CONTRACT_TEST_TEST_TYPE -import java.io.File fun ctrfSpecConfigsFrom( specmaticConfig: SpecmaticConfig, @@ -17,50 +13,11 @@ fun ctrfSpecConfigsFrom( val specConfigs = testResultRecords.map { it.specification.orEmpty() to it.testType }.map { (specification, testType) -> - val source = associatedSource(specification, specmaticConfig, testType) - CtrfSpecConfig( - serviceType = serviceType, - specType = specType, - specification = specification, - sourceProvider = source.provider.name, - repository = source.repository.orEmpty(), - branch = source.branch ?: "main", - ) + specmaticConfig.getCtrfSpecConfig(specification, testType, serviceType, specType) } return specConfigs } -private fun associatedSource(specification: String, specmaticConfig: SpecmaticConfig, testType: String): Source { - val specName = File(specification).name - val source = when (testType) { - CONTRACT_TEST_TEST_TYPE -> testSourceFromConfig(specName, specmaticConfig) - else -> stubSourceFromConfig(specName, specmaticConfig) - } - return source ?: Source() -} - -private fun testSourceFromConfig(specificationName: String, specmaticConfig: SpecmaticConfig): Source? { - val sources = SpecmaticConfig.getSources(specmaticConfig) - return sources.firstOrNull { source -> - source.test.orEmpty().any { test -> - test.contains(specificationName) - } - } -} - - -private fun stubSourceFromConfig(specificationName: String, specmaticConfig: SpecmaticConfig): Source? { - val sources = SpecmaticConfig.getSources(specmaticConfig) - return sources.firstOrNull { source -> - source.stub.orEmpty().any { stub -> - when (stub) { - is SpecExecutionConfig.StringValue -> stub.value.contains(specificationName) - is SpecExecutionConfig.ObjectValue -> stub.specs.any { it.contains(specificationName) } - } - } - } -} - From 22cf9ae82423c0240254f2ec14c3bb4c76cce1a7 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Wed, 3 Dec 2025 19:19:37 +0530 Subject: [PATCH 18/35] fix: bring back the coverage in SpecmaticAfterAllHook signature --- .../kotlin/io/specmatic/core/report/SpecmaticAfterAllHook.kt | 1 + core/src/main/kotlin/io/specmatic/stub/HttpStub.kt | 1 + .../src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt | 1 + 3 files changed, 3 insertions(+) diff --git a/core/src/main/kotlin/io/specmatic/core/report/SpecmaticAfterAllHook.kt b/core/src/main/kotlin/io/specmatic/core/report/SpecmaticAfterAllHook.kt index 4a57f29895..c5f6047be4 100644 --- a/core/src/main/kotlin/io/specmatic/core/report/SpecmaticAfterAllHook.kt +++ b/core/src/main/kotlin/io/specmatic/core/report/SpecmaticAfterAllHook.kt @@ -6,6 +6,7 @@ import io.specmatic.test.TestResultRecord interface SpecmaticAfterAllHook { fun generateReport( testResultRecords: List?, + coverage: Int, startTime: Long, endTime: Long, specConfigs: List, diff --git a/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt b/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt index dc2772f1a3..abd917df03 100644 --- a/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt +++ b/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt @@ -796,6 +796,7 @@ class HttpStub( hooks.forEach { it.generateReport( testResultRecords = ctrfTestResultRecords, + coverage = 0, startTime = startTime.toEpochMilli(), endTime = Instant.now().toEpochMilli(), specConfigs = ctrfSpecConfigsFrom(specmaticConfig, ctrfTestResultRecords), diff --git a/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt b/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt index b71207310d..3a4e36e70c 100644 --- a/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt +++ b/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt @@ -213,6 +213,7 @@ open class SpecmaticJUnitSupport { hooks.forEach { it.generateReport( testResultRecords = report.testResultRecords, + coverage = report.totalCoveragePercentage, startTime = start, endTime = end, specConfigs = specConfigs, From 95b184ec7f9ad643b53f77de71474a341a2d5465 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Wed, 3 Dec 2025 19:28:31 +0530 Subject: [PATCH 19/35] fix: use existing test result computation logic while saving it as a test result record --- core/src/main/kotlin/io/specmatic/stub/HttpStub.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt b/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt index abd917df03..58bc7b3df2 100644 --- a/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt +++ b/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt @@ -370,7 +370,7 @@ class HttpStub( responseStatus = httpResponse.status, request = httpRequest, response = httpResponse, - result = if(responseErrors.isEmpty()) TestResult.Success else TestResult.Failed, + result = httpLogMessage.toResult(), serviceType = "OPENAPI", requestContentType = httpRequest.headers["Content-Type"], specification = httpStubResponse.scenario?.specification, From 5d6ba77e20b0dcc45aa5889a33c757794ef91774 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Wed, 3 Dec 2025 19:37:32 +0530 Subject: [PATCH 20/35] fix: fix the values being saved in test result record --- core/src/main/kotlin/io/specmatic/stub/HttpStub.kt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt b/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt index 58bc7b3df2..8d6f6863d8 100644 --- a/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt +++ b/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt @@ -12,6 +12,7 @@ import io.ktor.server.request.* import io.ktor.server.response.* import io.ktor.util.* import io.ktor.util.pipeline.* +import io.specmatic.conversions.convertPathParameterStyle import io.specmatic.core.APPLICATION_NAME import io.specmatic.core.APPLICATION_NAME_LOWER_CASE import io.specmatic.core.Constants.Companion.ARTIFACTS_PATH @@ -365,14 +366,14 @@ class HttpStub( } val ctrfTestResultRecord = TestResultRecord( - path = httpRequest.path, - method = httpRequest.method.orEmpty(), - responseStatus = httpResponse.status, + path = convertPathParameterStyle(httpLogMessage.scenario?.path ?: httpRequest.path), + method = httpLogMessage.scenario?.method ?: httpRequest.method.orEmpty(), + responseStatus = httpLogMessage.scenario?.status ?: 0, request = httpRequest, response = httpResponse, result = httpLogMessage.toResult(), serviceType = "OPENAPI", - requestContentType = httpRequest.headers["Content-Type"], + requestContentType = httpLogMessage.scenario?.requestContentType ?: httpRequest.headers["Content-Type"], specification = httpStubResponse.scenario?.specification, testType = STUB_TEST_TYPE, actualResponseStatus = httpResponse.status From 29ee2b241431e7d5a689b155de52df7b0ee79a08 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Wed, 3 Dec 2025 19:46:34 +0530 Subject: [PATCH 21/35] chore: make coverage parameter nullable in specmatic after all hook --- .../kotlin/io/specmatic/core/report/SpecmaticAfterAllHook.kt | 4 ++-- core/src/main/kotlin/io/specmatic/stub/HttpStub.kt | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/main/kotlin/io/specmatic/core/report/SpecmaticAfterAllHook.kt b/core/src/main/kotlin/io/specmatic/core/report/SpecmaticAfterAllHook.kt index c5f6047be4..7a7368a009 100644 --- a/core/src/main/kotlin/io/specmatic/core/report/SpecmaticAfterAllHook.kt +++ b/core/src/main/kotlin/io/specmatic/core/report/SpecmaticAfterAllHook.kt @@ -6,10 +6,10 @@ import io.specmatic.test.TestResultRecord interface SpecmaticAfterAllHook { fun generateReport( testResultRecords: List?, - coverage: Int, startTime: Long, endTime: Long, specConfigs: List, - reportFilePath: String + reportFilePath: String, + coverage: Int? = null ) } \ No newline at end of file diff --git a/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt b/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt index 8d6f6863d8..0f4b2e3892 100644 --- a/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt +++ b/core/src/main/kotlin/io/specmatic/stub/HttpStub.kt @@ -797,7 +797,6 @@ class HttpStub( hooks.forEach { it.generateReport( testResultRecords = ctrfTestResultRecords, - coverage = 0, startTime = startTime.toEpochMilli(), endTime = Instant.now().toEpochMilli(), specConfigs = ctrfSpecConfigsFrom(specmaticConfig, ctrfTestResultRecords), From 647d030c26d81c85075b082b70816ffcde27f040 Mon Sep 17 00:00:00 2001 From: Joel Rosario Date: Wed, 3 Dec 2025 20:00:05 +0530 Subject: [PATCH 22/35] Cleaned up evaluation of result on a log message for improved clarity --- core/src/main/kotlin/io/specmatic/core/log/HttpLogMessage.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/kotlin/io/specmatic/core/log/HttpLogMessage.kt b/core/src/main/kotlin/io/specmatic/core/log/HttpLogMessage.kt index 7bb9955449..e7aac88570 100644 --- a/core/src/main/kotlin/io/specmatic/core/log/HttpLogMessage.kt +++ b/core/src/main/kotlin/io/specmatic/core/log/HttpLogMessage.kt @@ -125,7 +125,8 @@ data class HttpLogMessage( fun toResult(): TestResult { return when { - this.examplePath != null || this.scenario != null && response?.status !in invalidRequestStatuses -> TestResult.Success + this.examplePath != null -> TestResult.Success + this.scenario != null && response?.status !in invalidRequestStatuses -> TestResult.Success scenario == null -> TestResult.MissingInSpec else -> TestResult.Failed } From 9b110c012804bace657d3384a29b555952e6a173 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Wed, 3 Dec 2025 20:14:52 +0530 Subject: [PATCH 23/35] dummy commit --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 9342754087..1cb40d5ab7 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -2,7 +2,7 @@ import io.specmatic.gradle.extensions.RepoType plugins { id("io.specmatic.gradle") - id("base") + id("base") } allprojects { From f91c1439cf8780ed0164a7687d5808e0b6eb3afd Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Thu, 4 Dec 2025 13:50:04 +0530 Subject: [PATCH 24/35] fix: fix the support for recursive negative pattern generation --- .../core/pattern/AllNegativePatterns.kt | 13 ++-- .../specmatic/core/pattern/DeferredPattern.kt | 2 +- .../core/pattern/JSONObjectPatternTest.kt | 62 +++++++++++++++++++ 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/core/src/main/kotlin/io/specmatic/core/pattern/AllNegativePatterns.kt b/core/src/main/kotlin/io/specmatic/core/pattern/AllNegativePatterns.kt index 44f1821b76..b7d8fcbbeb 100644 --- a/core/src/main/kotlin/io/specmatic/core/pattern/AllNegativePatterns.kt +++ b/core/src/main/kotlin/io/specmatic/core/pattern/AllNegativePatterns.kt @@ -12,12 +12,13 @@ class AllNegativePatterns : NegativePatternsTemplate() { config: NegativePatternConfiguration ): Map>> { return patternMap.mapValues { (key, pattern) -> - val resolvedPattern = resolvedHop(pattern, resolver) - resolvedPattern.negativeBasedOn( - row.stepDownOneLevelInJSONHierarchy(withoutOptionality(key)), - resolver, - config - ) + resolver.withCyclePrevention(pattern, isOptional(key)) { cyclePreventedResolver -> + pattern.negativeBasedOn( + row.stepDownOneLevelInJSONHierarchy(withoutOptionality(key)), + cyclePreventedResolver, + config + ) + } ?: emptySequence() } } } \ No newline at end of file diff --git a/core/src/main/kotlin/io/specmatic/core/pattern/DeferredPattern.kt b/core/src/main/kotlin/io/specmatic/core/pattern/DeferredPattern.kt index d34e1d9b23..01baf6d627 100644 --- a/core/src/main/kotlin/io/specmatic/core/pattern/DeferredPattern.kt +++ b/core/src/main/kotlin/io/specmatic/core/pattern/DeferredPattern.kt @@ -85,7 +85,7 @@ data class DeferredPattern( } override fun negativeBasedOn(row: Row, resolver: Resolver, config: NegativePatternConfiguration): Sequence> { - return resolver.getPattern(pattern).negativeBasedOn(row, resolver) + return resolver.getPattern(pattern).negativeBasedOn(row, resolver, config) } override fun encompasses(otherPattern: Pattern, thisResolver: Resolver, otherResolver: Resolver, typeStack: TypeStack): Result { diff --git a/core/src/test/kotlin/io/specmatic/core/pattern/JSONObjectPatternTest.kt b/core/src/test/kotlin/io/specmatic/core/pattern/JSONObjectPatternTest.kt index b367e43e5e..aeed0a5f00 100644 --- a/core/src/test/kotlin/io/specmatic/core/pattern/JSONObjectPatternTest.kt +++ b/core/src/test/kotlin/io/specmatic/core/pattern/JSONObjectPatternTest.kt @@ -28,6 +28,68 @@ import java.util.function.Consumer import java.util.stream.Stream internal class JSONObjectPatternTest { + + @ParameterizedTest + @CsvSource( + "true", + "false" + ) + fun `should generate the negative patterns for a pattern where one of the optional keys has a reference to the parent object leading to a recursive cycle`(withDataTypeNegatives: Boolean) { + val pattern = JSONObjectPattern( + mapOf( + "children?" to DeferredPattern("(Recursive)") + ) + ) + + val resolver = Resolver( + newPatterns = mapOf( + "(Recursive)" to pattern + ), + isNegative = true, + generation = GenerativeTestsEnabled(positiveOnly = false) + ) + + val patterns = pattern.negativeBasedOn( + Row(), + resolver, + NegativePatternConfiguration(withDataTypeNegatives = withDataTypeNegatives) + ).toList() + + assertThat(patterns.size).isEqualTo(3) + assertDoesNotThrow { patterns.first().value.generate(resolver) } + } + + @ParameterizedTest + @CsvSource( + "true", + "false" + ) + fun `should throw exception while generating the negative patterns for a pattern where one of the mandatory keys has a reference to the parent object leading to a recursive cycle`(withDataTypeNegatives: Boolean) { + val pattern = JSONObjectPattern( + mapOf( + "children" to DeferredPattern("(Recursive)") + ) + ) + + val resolver = Resolver( + newPatterns = mapOf( + "(Recursive)" to pattern + ), + isNegative = true, + generation = GenerativeTestsEnabled(positiveOnly = false) + ) + + val exception = assertThrows { + pattern.negativeBasedOn( + Row(), + resolver, + NegativePatternConfiguration(withDataTypeNegatives = withDataTypeNegatives) + ).toList() + } + + assertThat(exception.message).contains("Invalid pattern cycle") + } + @Test fun `should filter out optional and extended keys from object and sub-objects`() { val pattern = parsedPattern("""{ From 1a391d5e7fb35ff69f9f59db1362620aecc4fa91 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Thu, 4 Dec 2025 15:51:04 +0530 Subject: [PATCH 25/35] chore: add default branch as main in ctrfspecconfig generation on test side --- .../src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt b/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt index 3a4e36e70c..95d2536909 100644 --- a/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt +++ b/junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt @@ -206,7 +206,7 @@ open class SpecmaticJUnitSupport { specification = it.specification.orEmpty(), sourceProvider = it.sourceProvider, repository = it.sourceRepository, - branch = it.sourceRepositoryBranch + branch = it.sourceRepositoryBranch ?: "main" ) } } From c82fe09d39ca1a5a885b86ba09ce35c4bdeafae2 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Thu, 4 Dec 2025 16:54:36 +0530 Subject: [PATCH 26/35] chore: add helper methods in specmatic config required for ctrf report generation --- .../io/specmatic/core/SpecmaticConfig.kt | 40 ++++++++++++++----- .../core/report/CtrfSpecConfigUtils.kt | 4 +- gradle.properties | 2 +- 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/core/src/main/kotlin/io/specmatic/core/SpecmaticConfig.kt b/core/src/main/kotlin/io/specmatic/core/SpecmaticConfig.kt index 519a1931a1..88073920f5 100644 --- a/core/src/main/kotlin/io/specmatic/core/SpecmaticConfig.kt +++ b/core/src/main/kotlin/io/specmatic/core/SpecmaticConfig.kt @@ -306,17 +306,21 @@ data class SpecmaticConfig( } @JsonIgnore - fun getCtrfSpecConfig(specification: String, testType: String, serviceType: String, specType: String): CtrfSpecConfig { - val specName = File(specification).name + fun getCtrfSpecConfig(absoluteSpecPath: String, testType: String, serviceType: String, specType: String): CtrfSpecConfig { val source = when (testType) { - CONTRACT_TEST_TEST_TYPE -> testSourceFromConfig(specName) - else -> stubSourceFromConfig(specName) + CONTRACT_TEST_TEST_TYPE -> testSourceFromConfig(absoluteSpecPath) + else -> stubSourceFromConfig(absoluteSpecPath) } ?: Source() + val specPathFromConfig = when(testType) { + CONTRACT_TEST_TEST_TYPE -> testSpecPathFromConfigFor(absoluteSpecPath) + else -> stubSpecPathFromConfigFor(absoluteSpecPath) + } + return CtrfSpecConfig( serviceType = serviceType, specType = specType, - specification = specification, + specification = specPathFromConfig.orEmpty(), sourceProvider = source.provider.name, repository = source.repository.orEmpty(), branch = source.branch ?: "main", @@ -686,21 +690,37 @@ data class SpecmaticConfig( } @JsonIgnore - private fun testSourceFromConfig(specificationName: String): Source? { + fun testSpecPathFromConfigFor(absoluteSpecPath: String): String? { + val source = testSourceFromConfig(absoluteSpecPath) ?: return null + return source.specsUsedAsTest().firstOrNull { + absoluteSpecPath.contains(it) + } + } + + @JsonIgnore + fun stubSpecPathFromConfigFor(absoluteSpecPath: String): String? { + val source = stubSourceFromConfig(absoluteSpecPath) ?: return null + return source.specsUsedAsStub().firstOrNull { + absoluteSpecPath.contains(it) + } + } + + @JsonIgnore + private fun testSourceFromConfig(absoluteSpecPath: String): Source? { return sources.firstOrNull { source -> source.test.orEmpty().any { test -> - test.contains(specificationName) + absoluteSpecPath.contains(test) } } } @JsonIgnore - private fun stubSourceFromConfig(specificationName: String): Source? { + private fun stubSourceFromConfig(absoluteSpecPath: String): Source? { return sources.firstOrNull { source -> source.stub.orEmpty().any { stub -> when (stub) { - is SpecExecutionConfig.StringValue -> stub.value.contains(specificationName) - is SpecExecutionConfig.ObjectValue -> stub.specs.any { it.contains(specificationName) } + is SpecExecutionConfig.StringValue -> absoluteSpecPath.contains(stub.value) + is SpecExecutionConfig.ObjectValue -> stub.specs.any { absoluteSpecPath.contains(it) } } } } diff --git a/core/src/main/kotlin/io/specmatic/core/report/CtrfSpecConfigUtils.kt b/core/src/main/kotlin/io/specmatic/core/report/CtrfSpecConfigUtils.kt index 1b3e8bd06a..27e8c411d3 100644 --- a/core/src/main/kotlin/io/specmatic/core/report/CtrfSpecConfigUtils.kt +++ b/core/src/main/kotlin/io/specmatic/core/report/CtrfSpecConfigUtils.kt @@ -12,8 +12,8 @@ fun ctrfSpecConfigsFrom( ): List { val specConfigs = testResultRecords.map { it.specification.orEmpty() to it.testType - }.map { (specification, testType) -> - specmaticConfig.getCtrfSpecConfig(specification, testType, serviceType, specType) + }.map { (absoluteSpecPath, testType) -> + specmaticConfig.getCtrfSpecConfig(absoluteSpecPath, testType, serviceType, specType) } return specConfigs } diff --git a/gradle.properties b/gradle.properties index 5d713b294c..ce1c0a69d5 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ group=io.specmatic version=2.32.2-SNAPSHOT specmaticGradlePluginVersion=0.13.8 -specmaticReporterVersion=0.1.8 +specmaticReporterVersion=0.1.9 kotlin.daemon.jvmargs=-Xmx1024m org.gradle.jvmargs=-Xmx1024m \ No newline at end of file From 7d907048989b8b5dadd4356dc6fe880611407b50 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Thu, 4 Dec 2025 18:12:15 +0530 Subject: [PATCH 27/35] chore: update specmatic-reporter version to 0.1.10 --- core/src/main/kotlin/io/specmatic/test/TestResultRecord.kt | 2 +- gradle.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/kotlin/io/specmatic/test/TestResultRecord.kt b/core/src/main/kotlin/io/specmatic/test/TestResultRecord.kt index cceff8c711..7e67a0e7ca 100644 --- a/core/src/main/kotlin/io/specmatic/test/TestResultRecord.kt +++ b/core/src/main/kotlin/io/specmatic/test/TestResultRecord.kt @@ -9,7 +9,7 @@ import io.specmatic.reporter.ctrf.model.CtrfTestResultRecord import io.specmatic.reporter.internal.dto.coverage.CoverageStatus import io.specmatic.reporter.internal.dto.spec.operation.APIOperation import io.specmatic.reporter.model.TestResult -import io.specmatic.reporter.spec.model.OpenAPIOperation +import io.specmatic.reporter.model.OpenAPIOperation import java.time.Duration import java.time.Instant diff --git a/gradle.properties b/gradle.properties index ce1c0a69d5..416ae2079a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ group=io.specmatic version=2.32.2-SNAPSHOT specmaticGradlePluginVersion=0.13.8 -specmaticReporterVersion=0.1.9 +specmaticReporterVersion=0.1.10 kotlin.daemon.jvmargs=-Xmx1024m org.gradle.jvmargs=-Xmx1024m \ No newline at end of file From 46d120d7f29034107c983ae59b5b7467c5dcc9a9 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Thu, 4 Dec 2025 14:37:35 +0530 Subject: [PATCH 28/35] Dictionary behavioral change to introduce leniency - Instead of calling out invalid-values and halting generation - Dictionary will now try to find the best random match and use it - If none found random data generation logic will be used --- .../conversions/BasicAuthSecurityScheme.kt | 2 +- .../kotlin/io/specmatic/core/Dictionary.kt | 46 ++++++++++++++++--- .../main/kotlin/io/specmatic/core/Resolver.kt | 8 +--- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/core/src/main/kotlin/io/specmatic/conversions/BasicAuthSecurityScheme.kt b/core/src/main/kotlin/io/specmatic/conversions/BasicAuthSecurityScheme.kt index 0d60c89257..3f16cb58fb 100644 --- a/core/src/main/kotlin/io/specmatic/conversions/BasicAuthSecurityScheme.kt +++ b/core/src/main/kotlin/io/specmatic/conversions/BasicAuthSecurityScheme.kt @@ -92,7 +92,7 @@ data class BasicAuthSecurityScheme(private val token: String? = null) : OpenAPIS private fun getTokenFromDictionary(resolver: Resolver): ReturnValue? { val updatedResolver = resolver.updateLookupForParam(BreadCrumb.HEADER.value) val dictionaryValue = updatedResolver.dictionary.getValueFor(AUTHORIZATION, StringPattern(), updatedResolver) - val authHeader = dictionaryValue?.unwrapOrContractException() ?: return null + val authHeader = dictionaryValue ?: return null if (authHeader !is StringValue) return HasFailure(Result.Failure( breadCrumb = BreadCrumb.HEADER.with(AUTHORIZATION), diff --git a/core/src/main/kotlin/io/specmatic/core/Dictionary.kt b/core/src/main/kotlin/io/specmatic/core/Dictionary.kt index e2cefe598c..5f7250e28f 100644 --- a/core/src/main/kotlin/io/specmatic/core/Dictionary.kt +++ b/core/src/main/kotlin/io/specmatic/core/Dictionary.kt @@ -50,13 +50,33 @@ data class Dictionary(private val data: Map, private val focusedD val resolved = resolvedHop(pattern, resolver) val lookupKey = withPatternDelimiters(resolved.typeName) val defaultValue = defaultData[lookupKey] ?: return null - return getReturnValueFor(lookupKey, defaultValue, resolved, resolver)?.withDefault(null) { it } + return getReturnValueFor(lookupKey, defaultValue, resolved, resolver)?.realise( + hasValue = { it, _ -> it }, + orFailure = { f -> + logger.debug(f.toFailure().reportString()) + null + }, + orException = { e -> + logger.debug(e.toHasFailure().toFailure().reportString()) + null + }, + ) } - fun getValueFor(lookup: String, pattern: Pattern, resolver: Resolver): ReturnValue? { + fun getValueFor(lookup: String, pattern: Pattern, resolver: Resolver): Value? { val tailEndKey = lookup.tailEndKey() val dictionaryValue = focusedData[tailEndKey] ?: return null - return getReturnValueFor(lookup, dictionaryValue, pattern, resolver) + return getReturnValueFor(lookup, dictionaryValue, pattern, resolver)?.realise( + hasValue = { it, _ -> it }, + orFailure = { f -> + logger.debug(f.toFailure().reportString()) + null + }, + orException = { e -> + logger.debug(e.toHasFailure().toFailure().reportString()) + null + }, + ) } private fun focusInto( @@ -73,7 +93,8 @@ data class Dictionary(private val data: Map, private val focusedD private fun getReturnValueFor(lookup: String, value: Value, pattern: Pattern, resolver: Resolver): ReturnValue? { val valueToMatch = getValueToMatch(value, pattern, resolver) ?: return null return runCatching { - val result = pattern.fillInTheBlanks(valueToMatch, resolver.copy(isNegative = false), removeExtraKeys = true) + val parsedValue = pattern.parse(valueToMatch.toUnformattedString(), resolver) + val result = pattern.fillInTheBlanks(parsedValue, resolver.copy(isNegative = false), removeExtraKeys = true) if (result is ReturnFailure && resolver.isNegative) return null result.addDetails("Invalid Dictionary value at \"$lookup\"", breadCrumb = "") }.getOrElse(::HasException) @@ -82,13 +103,13 @@ data class Dictionary(private val data: Map, private val focusedD private fun getValueToMatch(value: Value, pattern: Pattern, resolver: Resolver, overrideNestedCheck: Boolean = false): Value? { if (value !is JSONArrayValue) return value.takeIf { pattern.isScalar(resolver) || overrideNestedCheck } if (pattern !is SequenceType) { - return if (overrideNestedCheck) value else value.list.randomOrNull() + return if (overrideNestedCheck) value else selectValue(pattern, value.list, resolver) } val patternDepth = calculateDepth(pattern) { (resolvedHop(it, resolver) as? SequenceType)?.memberList?.patternList() } val valueDepth = calculateDepth(value) { (it as? JSONArrayValue)?.list } return when { - valueDepth > patternDepth -> value.list.randomOrNull() + valueDepth > patternDepth -> selectValue(pattern, value.list, resolver) else -> value } } @@ -105,11 +126,22 @@ data class Dictionary(private val data: Map, private val focusedD private fun Pattern.isScalar(resolver: Resolver): Boolean { val resolved = resolvedHop(this, resolver) - return resolved is ScalarType || resolved is URLPathSegmentPattern || resolved is QueryParameterScalarPattern + return resolved is ScalarType || resolved is URLPathSegmentPattern } private fun resetFocus(): Dictionary = copy(focusedData = emptyMap()) + private fun selectValue(pattern: Pattern, values: List, resolver: Resolver): Value? { + return values.shuffled().firstOrNull { value -> + runCatching { + pattern.matches(value, resolver).isSuccess() + }.getOrElse { e -> + logger.debug(e, "Failed to select value $values from dictionary for ${pattern.typeName}") + false + } + } + } + companion object { private const val SPECMATIC_CONSTANTS = "SPECMATIC_CONSTANTS" diff --git a/core/src/main/kotlin/io/specmatic/core/Resolver.kt b/core/src/main/kotlin/io/specmatic/core/Resolver.kt index 8796e5879b..82096e101c 100644 --- a/core/src/main/kotlin/io/specmatic/core/Resolver.kt +++ b/core/src/main/kotlin/io/specmatic/core/Resolver.kt @@ -218,7 +218,7 @@ data class Resolver( fun generate(pattern: Pattern): Value { val valueFromDict = dictionary.getValueFor(dictionaryLookupPath, pattern, this) if (valueFromDict != null) { - return valueFromDict.unwrapOrContractException() + return valueFromDict } val defaultValueFromDict = dictionary.getDefaultValueFor(pattern, this) @@ -322,11 +322,7 @@ data class Resolver( fun generateList(pattern: ListPattern): Value { val patternFocused = updateLookupPath(pattern.typeAlias) val valueFromDict = patternFocused.dictionary.getValueFor(patternFocused.dictionaryLookupPath, pattern, this) - if (valueFromDict != null) { - return valueFromDict.unwrapOrContractException() - } - - return this.updateLookupPath(pattern, pattern.pattern).generateRandomList(pattern.pattern) + return valueFromDict ?: this.updateLookupPath(pattern, pattern.pattern).generateRandomList(pattern.pattern) } private fun generateRandomList(pattern: Pattern): Value { From c54e8728b6358ddb3f8ad45726a4717441069d9d Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Thu, 4 Dec 2025 15:28:26 +0530 Subject: [PATCH 29/35] Add strictMode to Dictionary - Add tests for the same and lenient mode --- .../conversions/OpenApiSpecification.kt | 4 +- .../kotlin/io/specmatic/core/Dictionary.kt | 51 ++++---- .../main/kotlin/io/specmatic/core/Resolver.kt | 8 +- .../integration_tests/DictionaryTest.kt | 115 ++++++++++++++++++ .../specmatic/core/pattern/EnumPatternTest.kt | 2 +- 5 files changed, 154 insertions(+), 26 deletions(-) diff --git a/core/src/main/kotlin/io/specmatic/conversions/OpenApiSpecification.kt b/core/src/main/kotlin/io/specmatic/conversions/OpenApiSpecification.kt index 6e212c4de0..683c3d3641 100644 --- a/core/src/main/kotlin/io/specmatic/conversions/OpenApiSpecification.kt +++ b/core/src/main/kotlin/io/specmatic/conversions/OpenApiSpecification.kt @@ -268,9 +268,9 @@ class OpenApiSpecification( ) } - fun loadDictionary(openApiFilePath: String, dictionaryPathFromConfig: String?): Dictionary { + fun loadDictionary(openApiFilePath: String, dictionaryPathFromConfig: String?, strictMode: Boolean = false): Dictionary { val dictionaryFile = getDictionaryFile(File(openApiFilePath), dictionaryPathFromConfig) - return if (dictionaryFile != null) Dictionary.from(dictionaryFile) else Dictionary.empty() + return if (dictionaryFile != null) Dictionary.from(dictionaryFile, strictMode) else Dictionary.empty(strictMode) } private fun getDictionaryFile(openApiFile: File, dictionaryPathFromConfig: String?): File? { diff --git a/core/src/main/kotlin/io/specmatic/core/Dictionary.kt b/core/src/main/kotlin/io/specmatic/core/Dictionary.kt index 5f7250e28f..6cf168cb43 100644 --- a/core/src/main/kotlin/io/specmatic/core/Dictionary.kt +++ b/core/src/main/kotlin/io/specmatic/core/Dictionary.kt @@ -12,7 +12,11 @@ import io.specmatic.test.ExampleProcessor.toFactStore import io.specmatic.test.asserts.WILDCARD_INDEX import java.io.File -data class Dictionary(private val data: Map, private val focusedData: Map = emptyMap()) { +data class Dictionary( + private val data: Map, + private val focusedData: Map = emptyMap(), + private val strictMode: Boolean = false +) { private val defaultData: Map = data.filterKeys(::isPatternToken) fun plus(other: Map): Dictionary { @@ -53,12 +57,10 @@ data class Dictionary(private val data: Map, private val focusedD return getReturnValueFor(lookupKey, defaultValue, resolved, resolver)?.realise( hasValue = { it, _ -> it }, orFailure = { f -> - logger.debug(f.toFailure().reportString()) - null + logger.debug(f.toFailure().reportString()); null }, orException = { e -> - logger.debug(e.toHasFailure().toFailure().reportString()) - null + logger.debug(e.toHasFailure().toFailure().reportString()); null }, ) } @@ -69,12 +71,12 @@ data class Dictionary(private val data: Map, private val focusedD return getReturnValueFor(lookup, dictionaryValue, pattern, resolver)?.realise( hasValue = { it, _ -> it }, orFailure = { f -> - logger.debug(f.toFailure().reportString()) - null + if (strictMode) f.toFailure().throwOnFailure() + else logger.debug(f.toFailure().reportString()); null }, orException = { e -> - logger.debug(e.toHasFailure().toFailure().reportString()) - null + if (strictMode) e.toHasFailure().toFailure().throwOnFailure() + else logger.debug(e.toHasFailure().toFailure().reportString()); null }, ) } @@ -91,10 +93,9 @@ data class Dictionary(private val data: Map, private val focusedD } private fun getReturnValueFor(lookup: String, value: Value, pattern: Pattern, resolver: Resolver): ReturnValue? { - val valueToMatch = getValueToMatch(value, pattern, resolver) ?: return null return runCatching { - val parsedValue = pattern.parse(valueToMatch.toUnformattedString(), resolver) - val result = pattern.fillInTheBlanks(parsedValue, resolver.copy(isNegative = false), removeExtraKeys = true) + val valueToMatch = getValueToMatch(value, pattern, resolver) ?: return null + val result = pattern.fillInTheBlanks(valueToMatch, resolver.copy(isNegative = false), removeExtraKeys = true) if (result is ReturnFailure && resolver.isNegative) return null result.addDetails("Invalid Dictionary value at \"$lookup\"", breadCrumb = "") }.getOrElse(::HasException) @@ -132,11 +133,17 @@ data class Dictionary(private val data: Map, private val focusedD private fun resetFocus(): Dictionary = copy(focusedData = emptyMap()) private fun selectValue(pattern: Pattern, values: List, resolver: Resolver): Value? { + if (strictMode) return values.randomOrNull() return values.shuffled().firstOrNull { value -> runCatching { - pattern.matches(value, resolver).isSuccess() + val result = pattern.matches(value, resolver) + if (result is Result.Failure) { + logger.debug("Invalid value $value from dictionary for ${pattern.typeName}") + logger.debug(result.reportString()) + } + result.isSuccess() }.getOrElse { e -> - logger.debug(e, "Failed to select value $values from dictionary for ${pattern.typeName}") + logger.debug(e, "Failed to select value $value from dictionary for ${pattern.typeName}") false } } @@ -145,7 +152,7 @@ data class Dictionary(private val data: Map, private val focusedD companion object { private const val SPECMATIC_CONSTANTS = "SPECMATIC_CONSTANTS" - fun from(file: File): Dictionary { + fun from(file: File, strictMode: Boolean = false): Dictionary { if (!file.exists()) throw ContractException( breadCrumb = file.path, errorMessage = "Expected dictionary file at ${file.path}, but it does not exist" @@ -159,7 +166,7 @@ data class Dictionary(private val data: Map, private val focusedD return runCatching { logger.log("Using dictionary file ${file.path}") val dictionary = readValueAs(file).resolveConstants() - from(data = dictionary.jsonObject) + from(data = dictionary.jsonObject, strictMode) }.getOrElse { e -> logger.debug(e) throw ContractException( @@ -169,11 +176,11 @@ data class Dictionary(private val data: Map, private val focusedD } } - fun fromYaml(content: String): Dictionary { + fun fromYaml(content: String, strictMode: Boolean = false): Dictionary { return runCatching { val value = yamlStringToValue(content) if (value !is JSONObjectValue) throw ContractException("Expected dictionary file to be a YAML object") - from(value.jsonObject) + from(value.jsonObject, strictMode) }.getOrElse { e -> throw ContractException( breadCrumb = "Error while parsing YAML dictionary content", @@ -182,12 +189,12 @@ data class Dictionary(private val data: Map, private val focusedD } } - fun from(data: Map): Dictionary { - return Dictionary(data = data) + fun from(data: Map, strictMode: Boolean = false): Dictionary { + return Dictionary(data = data, strictMode = strictMode) } - fun empty(): Dictionary { - return Dictionary(data = emptyMap()) + fun empty(strictMode: Boolean = false): Dictionary { + return Dictionary(data = emptyMap(), strictMode = strictMode) } private fun JSONObjectValue.resolveConstants(): JSONObjectValue { diff --git a/core/src/main/kotlin/io/specmatic/core/Resolver.kt b/core/src/main/kotlin/io/specmatic/core/Resolver.kt index 82096e101c..606f4bb9ac 100644 --- a/core/src/main/kotlin/io/specmatic/core/Resolver.kt +++ b/core/src/main/kotlin/io/specmatic/core/Resolver.kt @@ -264,11 +264,17 @@ data class Resolver( focusIntoProperty(keyPattern, focusKey, this@Resolver) } - return this.copy( + val patternUpdatedResolver = this.copy( dictionaryLookupPath = lookupPath, lookupPathsSeenSoFar = lookupPathsSeenSoFar.plus(lookupPath), dictionary = keyFocused ) + + return if (keyWithPattern?.pattern?.typeAlias == null || builtInPatterns.contains(keyWithPattern.pattern.typeAlias)) { + patternUpdatedResolver + } else { + patternUpdatedResolver.updateLookupPath(keyWithPattern.pattern.typeAlias) + } } fun updateLookupPath(pattern: T, childPattern: Pattern): Resolver where T: Pattern, T: SequenceType { diff --git a/core/src/test/kotlin/integration_tests/DictionaryTest.kt b/core/src/test/kotlin/integration_tests/DictionaryTest.kt index 307cb082ab..f11eb4fe74 100644 --- a/core/src/test/kotlin/integration_tests/DictionaryTest.kt +++ b/core/src/test/kotlin/integration_tests/DictionaryTest.kt @@ -14,6 +14,10 @@ import io.specmatic.core.SPECMATIC_STUB_DICTIONARY import io.specmatic.core.Scenario import io.specmatic.core.ScenarioInfo import io.specmatic.core.buildHttpPathPattern +import io.specmatic.core.log.Verbose +import io.specmatic.core.log.withLogger +import io.specmatic.core.pattern.AnyPattern +import io.specmatic.core.pattern.BooleanPattern import io.specmatic.core.pattern.ContractException import io.specmatic.core.pattern.EmailPattern import io.specmatic.core.pattern.JSONObjectPattern @@ -27,6 +31,7 @@ import io.specmatic.core.pattern.parsedJSON import io.specmatic.core.pattern.parsedJSONArray import io.specmatic.core.pattern.parsedJSONObject import io.specmatic.core.pattern.parsedPattern +import io.specmatic.core.utilities.toValue import io.specmatic.core.value.BooleanValue import io.specmatic.core.value.JSONArrayValue import io.specmatic.core.value.JSONObjectValue @@ -35,6 +40,7 @@ import io.specmatic.core.value.NumberValue import io.specmatic.core.value.StringValue import io.specmatic.stub.HttpStub import io.specmatic.stub.SPECMATIC_RESPONSE_CODE_HEADER +import io.specmatic.stub.captureStandardOutput import io.specmatic.stub.createStubFromContracts import io.specmatic.stub.httpRequestLog import io.specmatic.stub.httpResponseLog @@ -912,6 +918,115 @@ class DictionaryTest { } } + @Nested + inner class ConflictingValuesTest { + @Test + fun `should pick randomly matching value when there are conflicts in the dictionary path causing multiple types to exist`() { + val dictionary = "'*': { commonKey: [10, Twenty, specmatic@test.io, false] } ".let(Dictionary::fromYaml) + val patterns = listOf( + NumberPattern() to 10, + EmailPattern() to "specmatic@test.io", + BooleanPattern() to false + ).map { it.first to toValue(it.second) } + + assertThat(patterns).allSatisfy { (pattern, expectedValue) -> + val pattern = JSONObjectPattern(mapOf("commonKey" to pattern)) + val resolver = Resolver(dictionary = dictionary) + val value = pattern.generate(resolver).jsonObject.getValue("commonKey") + assertThat(value).isEqualTo(expectedValue) + } + } + + @Test + fun `should pick randomly matching value when there are conflicts in the dictionary path under the same schema`() { + val dictionary = "Schema: { commonKey: [10, Twenty, specmatic@test.io, false] } ".let(Dictionary::fromYaml) + val patterns = listOf( + NumberPattern() to 10, + EmailPattern() to "specmatic@test.io", + BooleanPattern() to false + ).map { it.first to toValue(it.second) } + + assertThat(patterns).allSatisfy { (pattern, expectedValue) -> + val pattern = JSONObjectPattern(mapOf("commonKey" to pattern), typeAlias = "(Schema)") + val resolver = Resolver(dictionary = dictionary) + val value = pattern.generate(resolver).jsonObject.getValue("commonKey") + assertThat(value).isEqualTo(expectedValue) + } + } + + @Test + fun `should work with composite type multi-values`() { + val dictionary = """ + Schema: + commonKey: + - objectKey: ObjectValue + - - arrayKey: FirstArrayValue + - arrayKey: SecondArrayValue + """.let(Dictionary::fromYaml) + + val pattern = JSONObjectPattern(mapOf( + "commonKey" to AnyPattern( + extensions = emptyMap(), + pattern = listOf( + JSONObjectPattern(mapOf("objectKey" to StringPattern())), + ListPattern(JSONObjectPattern(mapOf("arrayKey" to StringPattern()))) + ), + ) + ), typeAlias = "(Schema)") + + val resolver = Resolver(dictionary = dictionary) + val value = pattern.generate(resolver) + val commonKeyValue = value.jsonObject.getValue("commonKey") + assertThat(commonKeyValue).satisfiesAnyOf( + { + assertThat(it).isInstanceOf(JSONObjectValue::class.java); it as JSONObjectValue + assertThat(it.jsonObject.getValue("objectKey")).isEqualTo(StringValue("ObjectValue")) + }, + { + assertThat(it).isInstanceOf(JSONArrayValue::class.java) ; it as JSONArrayValue + assertThat(it.list).hasSize(2).containsExactlyInAnyOrder( + JSONObjectValue(mapOf("arrayKey" to StringValue("FirstArrayValue"))), + JSONObjectValue(mapOf("arrayKey" to StringValue("SecondArrayValue"))) + ) + } + ) + } + + @Test + fun `should log all tried values when resolving best matching dictionary value`() { + val dictionary = "'*': { commonKey: [10, Twenty, specmatic@test.io] } ".let(Dictionary::fromYaml) + val pattern = JSONObjectPattern(mapOf("commonKey" to BooleanPattern())) + val resolver = Resolver(dictionary = dictionary) + val (stdout, value) = captureStandardOutput { + withLogger(Verbose()) { pattern.generate(resolver) } + } + + val commonKeyValue = value.jsonObject.getValue("commonKey") + assertThat(commonKeyValue).isInstanceOf(BooleanValue::class.java) + assertThat(stdout).containsIgnoringWhitespaces(""" + Invalid value Twenty from dictionary for boolean + Expected boolean, actual was "Twenty" + Invalid value specmatic@test.io from dictionary for boolean + Expected boolean, actual was "specmatic@test.io" + Invalid value 10 from dictionary for boolean + Expected boolean, actual was 10 (number) + """.trimIndent()) + } + + @Test + fun `should result in an exception if the value picked is invalid in strict-mode`() { + val dictionary = "'*': { commonKey: [10, Twenty, specmatic@test.io] } ".let(Dictionary::fromYaml) + val pattern = JSONObjectPattern(mapOf("commonKey" to BooleanPattern())) + val resolver = Resolver(dictionary = dictionary.copy(strictMode = true)) + val exception = assertThrows { pattern.generate(resolver) } + + assertThat(exception.report()).containsIgnoringWhitespaces(""" + >> commonKey + Invalid Dictionary value at ".commonKey" + """.trimIndent()) + } + } + companion object { @JvmStatic fun listPatternToSingleValueProvider(): Stream { diff --git a/core/src/test/kotlin/io/specmatic/core/pattern/EnumPatternTest.kt b/core/src/test/kotlin/io/specmatic/core/pattern/EnumPatternTest.kt index 410a768a00..63fe806b8c 100644 --- a/core/src/test/kotlin/io/specmatic/core/pattern/EnumPatternTest.kt +++ b/core/src/test/kotlin/io/specmatic/core/pattern/EnumPatternTest.kt @@ -185,7 +185,7 @@ class EnumPatternTest { val enumPattern = EnumPattern(enumValues, typeAlias = "(AnimalType)") val jsonPattern = JSONObjectPattern(mapOf("type" to enumPattern), typeAlias = "(Test)") - val dictionary = "Test: { type: Dog }".let(Dictionary::fromYaml) + val dictionary = "AnimalType: Dog".let(Dictionary::fromYaml) val resolver = Resolver(newPatterns = mapOf("(AnimalType)" to enumPattern), dictionary = dictionary) val value = JSONObjectValue(mapOf("type" to StringValue("(AnimalType)"))) val filledInValue = jsonPattern.fillInTheBlanks(value, resolver).value From 0b0b5327bfaf0307814c846bebaf919569cd8a6e Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Thu, 4 Dec 2025 16:48:54 +0530 Subject: [PATCH 30/35] Resolve failing tests for lenient dictionary - Make the value selection lenient using keyChecks --- .../conversions/OpenApiSpecification.kt | 4 +- .../kotlin/io/specmatic/core/Dictionary.kt | 3 +- core/src/main/kotlin/io/specmatic/stub/api.kt | 1 + .../integration_tests/DictionaryTest.kt | 47 ++++++++++--------- .../integration_tests/PartialExampleTest.kt | 3 ++ .../core/pattern/JSONObjectPatternTest.kt | 8 ++-- 6 files changed, 38 insertions(+), 28 deletions(-) diff --git a/core/src/main/kotlin/io/specmatic/conversions/OpenApiSpecification.kt b/core/src/main/kotlin/io/specmatic/conversions/OpenApiSpecification.kt index 683c3d3641..c628e30dd4 100644 --- a/core/src/main/kotlin/io/specmatic/conversions/OpenApiSpecification.kt +++ b/core/src/main/kotlin/io/specmatic/conversions/OpenApiSpecification.kt @@ -88,8 +88,8 @@ class OpenApiSpecification( private val specificationPath: String? = null, private val securityConfiguration: SecurityConfiguration? = null, private val specmaticConfig: SpecmaticConfig = SpecmaticConfig(), - private val dictionary: Dictionary = loadDictionary(openApiFilePath, specmaticConfig.getStubDictionary()), - private val strictMode: Boolean = false + private val strictMode: Boolean = false, + private val dictionary: Dictionary = loadDictionary(openApiFilePath, specmaticConfig.getStubDictionary(), strictMode), ) : IncludedSpecification, ApiSpecification { init { StringProviders // Trigger early initialization of StringProviders to ensure all providers are loaded at startup diff --git a/core/src/main/kotlin/io/specmatic/core/Dictionary.kt b/core/src/main/kotlin/io/specmatic/core/Dictionary.kt index 6cf168cb43..c73003ce3e 100644 --- a/core/src/main/kotlin/io/specmatic/core/Dictionary.kt +++ b/core/src/main/kotlin/io/specmatic/core/Dictionary.kt @@ -136,7 +136,7 @@ data class Dictionary( if (strictMode) return values.randomOrNull() return values.shuffled().firstOrNull { value -> runCatching { - val result = pattern.matches(value, resolver) + val result = pattern.matches(value, resolver.copy(findKeyErrorCheck = noPatternKeyCheckDictionary)) if (result is Result.Failure) { logger.debug("Invalid value $value from dictionary for ${pattern.typeName}") logger.debug(result.reportString()) @@ -151,6 +151,7 @@ data class Dictionary( companion object { private const val SPECMATIC_CONSTANTS = "SPECMATIC_CONSTANTS" + private val noPatternKeyCheckDictionary = KeyCheck(noPatternKeyCheck, IgnoreUnexpectedKeys) fun from(file: File, strictMode: Boolean = false): Dictionary { if (!file.exists()) throw ContractException( diff --git a/core/src/main/kotlin/io/specmatic/stub/api.kt b/core/src/main/kotlin/io/specmatic/stub/api.kt index 59913e969f..55c0016454 100644 --- a/core/src/main/kotlin/io/specmatic/stub/api.kt +++ b/core/src/main/kotlin/io/specmatic/stub/api.kt @@ -1063,6 +1063,7 @@ fun loadIfSupportedAPISpecification( contractPathData.repository, contractPathData.branch, contractPathData.specificationPath, + strictMode = specmaticConfig.getStubStrictMode() ?: false ).copy(specmaticConfig = specmaticConfig), ) } catch (e: Throwable) { diff --git a/core/src/test/kotlin/integration_tests/DictionaryTest.kt b/core/src/test/kotlin/integration_tests/DictionaryTest.kt index f11eb4fe74..891c9837e0 100644 --- a/core/src/test/kotlin/integration_tests/DictionaryTest.kt +++ b/core/src/test/kotlin/integration_tests/DictionaryTest.kt @@ -19,6 +19,7 @@ import io.specmatic.core.log.withLogger import io.specmatic.core.pattern.AnyPattern import io.specmatic.core.pattern.BooleanPattern import io.specmatic.core.pattern.ContractException +import io.specmatic.core.pattern.DeferredPattern import io.specmatic.core.pattern.EmailPattern import io.specmatic.core.pattern.JSONObjectPattern import io.specmatic.core.pattern.ListPattern @@ -592,6 +593,25 @@ class DictionaryTest { } } + @Test + fun `if subKey has typeAlias should focus into subKey schema in dictionary instead of staying on pattern schema`() { + val dictionary = """ + '*': + error: Invalid-Request-Try-Again + ErrorSchema: + status: 400 + """.trimIndent().let(Dictionary::fromYaml) + val pattern = JSONObjectPattern(mapOf("error" to DeferredPattern("(ErrorSchema)"))) + val errorPattern = JSONObjectPattern(mapOf("status" to NumberPattern())) + + val resolver = Resolver(dictionary = dictionary, newPatterns = mapOf("(ErrorSchema)" to errorPattern)) + val generatedValue = pattern.generate(resolver) + val errorValue = generatedValue.jsonObject.getValue("error") + + assertThat(errorValue).isInstanceOf(JSONObjectValue::class.java); errorValue as JSONObjectValue + assertThat(errorValue.jsonObject.getValue("status")).isEqualTo(NumberValue(400)) + } + @Nested inner class MultiValueDictionaryTests { @@ -621,10 +641,10 @@ class DictionaryTest { } @Test - fun `should throw an exception when array key contains invalid value and pattern is an array`() { + fun `should throw an exception when array key contains invalid value and pattern is an array with strict mode`() { val dictionary = "Schema: { array: [1, abc, 3] }".let(Dictionary::fromYaml) val pattern = JSONObjectPattern(mapOf("array" to ListPattern(NumberPattern())), typeAlias = "(Schema)") - val resolver = Resolver(dictionary = dictionary) + val resolver = Resolver(dictionary = dictionary.copy(strictMode = true)) val exception = assertThrows { pattern.generate(resolver) } assertThat(exception.report()).isEqualToNormalizingWhitespace(""" @@ -1006,8 +1026,12 @@ class DictionaryTest { assertThat(stdout).containsIgnoringWhitespaces(""" Invalid value Twenty from dictionary for boolean Expected boolean, actual was "Twenty" + """.trimIndent()) + assertThat(stdout).containsIgnoringWhitespaces(""" Invalid value specmatic@test.io from dictionary for boolean Expected boolean, actual was "specmatic@test.io" + """.trimIndent()) + assertThat(stdout).containsIgnoringWhitespaces(""" Invalid value 10 from dictionary for boolean Expected boolean, actual was 10 (number) """.trimIndent()) @@ -1061,25 +1085,6 @@ class DictionaryTest { // List[Pattern] Arguments.of( listPatternOf(NumberPattern()), parsedJSONArray("""[[1, 2], [3, 4]]""") - ), - Arguments.of( - listPatternOf(NumberPattern()), parsedJSONArray("""[[], [3, 4]]""") - ), - // List[List[Pattern]] - Arguments.of( - listPatternOf(NumberPattern(), nestedLevel = 1), parsedJSONArray("""[[[1, 2]], [[3, 4]]]""") - ), - Arguments.of( - listPatternOf(NumberPattern(), nestedLevel = 1), parsedJSONArray("""[[[1, 2]], [[]]]""") - ), - // List[List[List[Pattern]]] - Arguments.of( - listPatternOf(NumberPattern(), nestedLevel = 2), - parsedJSONArray("""[[[[1, 2]], [[3, 4]]], [[[5, 6]], [[7, 8]]]]""") - ), - Arguments.of( - listPatternOf(NumberPattern(), nestedLevel = 2), - parsedJSONArray("""[[[[1, 2]], [[3, 4]]], [[[]], [[7, 8]]]]""") ) ) } diff --git a/core/src/test/kotlin/integration_tests/PartialExampleTest.kt b/core/src/test/kotlin/integration_tests/PartialExampleTest.kt index 1622b16e4b..d18a9c1638 100644 --- a/core/src/test/kotlin/integration_tests/PartialExampleTest.kt +++ b/core/src/test/kotlin/integration_tests/PartialExampleTest.kt @@ -6,6 +6,7 @@ import io.specmatic.core.SPECMATIC_STUB_DICTIONARY import io.specmatic.core.log.DebugLogger import io.specmatic.core.log.withLogger import io.specmatic.core.pattern.parsedJSONObject +import io.specmatic.core.utilities.Flags.Companion.STUB_STRICT_MODE import io.specmatic.core.value.JSONObjectValue import io.specmatic.core.value.NumberValue import io.specmatic.core.value.StringValue @@ -448,6 +449,7 @@ class PartialExampleTest { fun `partial example using invalid dictionary should throw an error at runtime`() { try { System.setProperty(SPECMATIC_STUB_DICTIONARY, "src/test/resources/openapi/substitutions/dictionary.json") + System.setProperty(STUB_STRICT_MODE, "true") createStubFromContracts( listOf("src/test/resources/openapi/substitutions/partial_with_invalid_dictionary_value.yaml"), @@ -466,6 +468,7 @@ class PartialExampleTest { } } finally { System.clearProperty(SPECMATIC_STUB_DICTIONARY) + System.clearProperty(STUB_STRICT_MODE) } } diff --git a/core/src/test/kotlin/io/specmatic/core/pattern/JSONObjectPatternTest.kt b/core/src/test/kotlin/io/specmatic/core/pattern/JSONObjectPatternTest.kt index aeed0a5f00..5b870b6e3f 100644 --- a/core/src/test/kotlin/io/specmatic/core/pattern/JSONObjectPatternTest.kt +++ b/core/src/test/kotlin/io/specmatic/core/pattern/JSONObjectPatternTest.kt @@ -893,7 +893,7 @@ internal class JSONObjectPatternTest { } @Test - fun `should throw an exception when dictionary values in an array do not match`() { + fun `should throw an exception when dictionary values in an array do not match with strict mode`() { val personTypeAlias = "(Person)" val personPattern = JSONObjectPattern( @@ -908,7 +908,7 @@ internal class JSONObjectPatternTest { val resolver = Resolver( newPatterns = mapOf(personTypeAlias to personPattern), - dictionary = dictionary + dictionary = dictionary.copy(strictMode = true) ) val exception = assertThrows { @@ -1080,7 +1080,7 @@ internal class JSONObjectPatternTest { } @Test - fun `throw exception when example is found but invalid `() { + fun `throw exception when example is found but invalid with strict mode`() { val personTypeAlias = "(Person)" val personPattern = JSONObjectPattern( @@ -1095,7 +1095,7 @@ internal class JSONObjectPatternTest { val dictionary = "Person: { id: $id }".let(Dictionary::fromYaml) val resolver = Resolver( newPatterns = mapOf(personTypeAlias to personPattern), - dictionary = dictionary + dictionary = dictionary.copy(strictMode = true) ) assertThatThrownBy { From d672556fdfb218f0f968b79729443ef34d1342f8 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Thu, 4 Dec 2025 17:09:22 +0530 Subject: [PATCH 31/35] Change leniency of strictMode in Dictionary --- .../src/main/kotlin/io/specmatic/core/Dictionary.kt | 13 ++++++++++++- .../test/kotlin/integration_tests/DictionaryTest.kt | 5 ++++- .../specmatic/core/pattern/JSONObjectPatternTest.kt | 8 +++++--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/core/src/main/kotlin/io/specmatic/core/Dictionary.kt b/core/src/main/kotlin/io/specmatic/core/Dictionary.kt index c73003ce3e..d137cff0bd 100644 --- a/core/src/main/kotlin/io/specmatic/core/Dictionary.kt +++ b/core/src/main/kotlin/io/specmatic/core/Dictionary.kt @@ -133,7 +133,18 @@ data class Dictionary( private fun resetFocus(): Dictionary = copy(focusedData = emptyMap()) private fun selectValue(pattern: Pattern, values: List, resolver: Resolver): Value? { - if (strictMode) return values.randomOrNull() + if (!strictMode) return selectValueLenient(pattern, values, resolver) + return selectValueLenient(pattern, values, resolver) ?: throw ContractException( + errorMessage = """ + None of the dictionary values matched the schema. + This could happen due to conflicts in the dictionary at the same json path, due to conflicting dataTypes at the same json path between multiple payloads + strictMode enforces the presence of matching values in the dictionary if the json-path is present + Either ensure that a matching value exists in the dictionary or disable strictMode + """.trimIndent() + ) + } + + private fun selectValueLenient(pattern: Pattern, values: List, resolver: Resolver): Value? { return values.shuffled().firstOrNull { value -> runCatching { val result = pattern.matches(value, resolver.copy(findKeyErrorCheck = noPatternKeyCheckDictionary)) diff --git a/core/src/test/kotlin/integration_tests/DictionaryTest.kt b/core/src/test/kotlin/integration_tests/DictionaryTest.kt index 891c9837e0..fbc72d345b 100644 --- a/core/src/test/kotlin/integration_tests/DictionaryTest.kt +++ b/core/src/test/kotlin/integration_tests/DictionaryTest.kt @@ -1046,7 +1046,10 @@ class DictionaryTest { assertThat(exception.report()).containsIgnoringWhitespaces(""" >> commonKey - Invalid Dictionary value at ".commonKey" + None of the dictionary values matched the schema. + This could happen due to conflicts in the dictionary at the same json path, due to conflicting dataTypes at the same json path between multiple payloads + strictMode enforces the presence of matching values in the dictionary if the json-path is present + Either ensure that a matching value exists in the dictionary or disable strictMode """.trimIndent()) } } diff --git a/core/src/test/kotlin/io/specmatic/core/pattern/JSONObjectPatternTest.kt b/core/src/test/kotlin/io/specmatic/core/pattern/JSONObjectPatternTest.kt index 5b870b6e3f..a2306bd0f2 100644 --- a/core/src/test/kotlin/io/specmatic/core/pattern/JSONObjectPatternTest.kt +++ b/core/src/test/kotlin/io/specmatic/core/pattern/JSONObjectPatternTest.kt @@ -916,9 +916,11 @@ internal class JSONObjectPatternTest { } assertThat(exception.report()).isEqualToNormalizingWhitespace(""" - >> addresses[0] - Invalid Dictionary value at "Person.addresses" - Expected string, actual was 10 (number) + >> addresses + None of the dictionary values matched the schema. + This could happen due to conflicts in the dictionary at the same json path, due to conflicting dataTypes at the same json path between multiple payloads + strictMode enforces the presence of matching values in the dictionary if the json-path is present + Either ensure that a matching value exists in the dictionary or disable strictMode """.trimIndent()) } From 7007916392be1c0fef1da90791e69177a0d39023 Mon Sep 17 00:00:00 2001 From: Joel Rosario Date: Thu, 4 Dec 2025 17:54:09 +0530 Subject: [PATCH 32/35] Minor refactor of Dictionary.selectValue for clarity --- .../kotlin/io/specmatic/core/Dictionary.kt | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/core/src/main/kotlin/io/specmatic/core/Dictionary.kt b/core/src/main/kotlin/io/specmatic/core/Dictionary.kt index d137cff0bd..397a5fa77f 100644 --- a/core/src/main/kotlin/io/specmatic/core/Dictionary.kt +++ b/core/src/main/kotlin/io/specmatic/core/Dictionary.kt @@ -133,15 +133,21 @@ data class Dictionary( private fun resetFocus(): Dictionary = copy(focusedData = emptyMap()) private fun selectValue(pattern: Pattern, values: List, resolver: Resolver): Value? { - if (!strictMode) return selectValueLenient(pattern, values, resolver) - return selectValueLenient(pattern, values, resolver) ?: throw ContractException( - errorMessage = """ - None of the dictionary values matched the schema. - This could happen due to conflicts in the dictionary at the same json path, due to conflicting dataTypes at the same json path between multiple payloads - strictMode enforces the presence of matching values in the dictionary if the json-path is present - Either ensure that a matching value exists in the dictionary or disable strictMode - """.trimIndent() - ) + val lenientlySelectedValue = selectValueLenient(pattern, values, resolver) + + if (strictMode && lenientlySelectedValue == null) { + throw ContractException( + errorMessage = + """ + None of the dictionary values matched the schema. + This could happen due to conflicts in the dictionary at the same json path, due to conflicting dataTypes at the same json path between multiple payloads + strictMode enforces the presence of matching values in the dictionary if the json-path is present + Either ensure that a matching value exists in the dictionary or disable strictMode + """.trimIndent(), + ) + } + + return lenientlySelectedValue } private fun selectValueLenient(pattern: Pattern, values: List, resolver: Resolver): Value? { From 61f3164d71addccdb01ce2073118abeba0d13ee3 Mon Sep 17 00:00:00 2001 From: Joel Rosario Date: Thu, 4 Dec 2025 18:57:07 +0530 Subject: [PATCH 33/35] Debug log improvements when a value is not matched in the dictionary --- .../kotlin/io/specmatic/core/Dictionary.kt | 29 +++++++++++++++++-- .../integration_tests/DictionaryTest.kt | 12 ++++---- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/core/src/main/kotlin/io/specmatic/core/Dictionary.kt b/core/src/main/kotlin/io/specmatic/core/Dictionary.kt index 397a5fa77f..ffb6827662 100644 --- a/core/src/main/kotlin/io/specmatic/core/Dictionary.kt +++ b/core/src/main/kotlin/io/specmatic/core/Dictionary.kt @@ -150,17 +150,42 @@ data class Dictionary( return lenientlySelectedValue } + private val dictionaryMismatchMessages = object : MismatchMessages { + override fun mismatchMessage(expected: String, actual: String): String { + return "Expected $expected but got $actual in the dictionary" + } + + override fun unexpectedKey(keyLabel: String, keyName: String): String { + return "Unexpected $keyLabel $keyName in the dictionary" + } + + override fun expectedKeyWasMissing(keyLabel: String, keyName: String): String { + return "Expected $keyLabel $keyName was missing in the dictionary" + } + } + private fun selectValueLenient(pattern: Pattern, values: List, resolver: Resolver): Value? { + val dictionaryBreadcrumbs = ">> DICTIONARY.${resolver.dictionaryLookupPath}" + + val updatedResolver = resolver.copy( + findKeyErrorCheck = noPatternKeyCheckDictionary, + mismatchMessages = dictionaryMismatchMessages, + ) + return values.shuffled().firstOrNull { value -> runCatching { - val result = pattern.matches(value, resolver.copy(findKeyErrorCheck = noPatternKeyCheckDictionary)) + val result = pattern.matches(value, updatedResolver) + if (result is Result.Failure) { - logger.debug("Invalid value $value from dictionary for ${pattern.typeName}") + logger.debug(dictionaryBreadcrumbs) logger.debug(result.reportString()) + logger.debug(System.lineSeparator()) } result.isSuccess() }.getOrElse { e -> + logger.debug(">> DICTIONARY.${resolver.dictionaryLookupPath}") logger.debug(e, "Failed to select value $value from dictionary for ${pattern.typeName}") + logger.debug(System.lineSeparator()) false } } diff --git a/core/src/test/kotlin/integration_tests/DictionaryTest.kt b/core/src/test/kotlin/integration_tests/DictionaryTest.kt index fbc72d345b..91a22646c1 100644 --- a/core/src/test/kotlin/integration_tests/DictionaryTest.kt +++ b/core/src/test/kotlin/integration_tests/DictionaryTest.kt @@ -1024,16 +1024,16 @@ class DictionaryTest { val commonKeyValue = value.jsonObject.getValue("commonKey") assertThat(commonKeyValue).isInstanceOf(BooleanValue::class.java) assertThat(stdout).containsIgnoringWhitespaces(""" - Invalid value Twenty from dictionary for boolean - Expected boolean, actual was "Twenty" + >> DICTIONARY..commonKey + Expected boolean but got "Twenty" in the dictionary """.trimIndent()) assertThat(stdout).containsIgnoringWhitespaces(""" - Invalid value specmatic@test.io from dictionary for boolean - Expected boolean, actual was "specmatic@test.io" + >> DICTIONARY..commonKey + Expected boolean but got 10 (number) in the dictionary """.trimIndent()) assertThat(stdout).containsIgnoringWhitespaces(""" - Invalid value 10 from dictionary for boolean - Expected boolean, actual was 10 (number) + >> DICTIONARY..commonKey + Expected boolean but got "specmatic@test.io" in the dictionary """.trimIndent()) } From 21647b59b937f0eafad5117efb2e8e64158c35e1 Mon Sep 17 00:00:00 2001 From: Joel Rosario Date: Thu, 4 Dec 2025 20:01:03 +0530 Subject: [PATCH 34/35] [ci skip] Minor version bump to 2.33.0 --- gradle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index 416ae2079a..878c417bd4 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ group=io.specmatic -version=2.32.2-SNAPSHOT +version=2.33.0-SNAPSHOT specmaticGradlePluginVersion=0.13.8 specmaticReporterVersion=0.1.10 kotlin.daemon.jvmargs=-Xmx1024m -org.gradle.jvmargs=-Xmx1024m \ No newline at end of file +org.gradle.jvmargs=-Xmx1024m From 0529dcb7492c1168fdad69d64d2b7dbb0deae8e8 Mon Sep 17 00:00:00 2001 From: Specmatic GitHub Service Account Date: Thu, 4 Dec 2025 14:38:08 +0000 Subject: [PATCH 35/35] chore(release): pre-release bump version 2.33.0 --- gradle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index 878c417bd4..178e96e094 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ group=io.specmatic -version=2.33.0-SNAPSHOT +version=2.33.0 specmaticGradlePluginVersion=0.13.8 specmaticReporterVersion=0.1.10 kotlin.daemon.jvmargs=-Xmx1024m -org.gradle.jvmargs=-Xmx1024m +org.gradle.jvmargs=-Xmx1024m \ No newline at end of file