diff --git a/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/aiclassification/AIModelsCheck.kt b/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/aiclassification/AIModelsCheck.kt index cbeae79ae3..b4769b490a 100644 --- a/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/aiclassification/AIModelsCheck.kt +++ b/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/aiclassification/AIModelsCheck.kt @@ -55,6 +55,8 @@ class AIModelsCheck : IntegrationTestRestBase() { val metricType = "TIME_WINDOW" val warmUpRep = 10 val maxAttemptRepair = 100 // i.e., the classifier has 10 times the chances to pick an action with non-400 response + val repairThreshold = 0.5 + val weaknessThreshold = 0.6 val runIterations = 500 val saveReport = false @@ -82,6 +84,9 @@ class AIModelsCheck : IntegrationTestRestBase() { EMConfig.AIClassificationMetrics.valueOf(metricType) config.aiResponseClassifierWarmup = warmUpRep config.maxRepairAttemptsInResponseClassification = maxAttemptRepair + config.classificationRepairThreshold = repairThreshold + config.aIResponseClassifierWeaknessThreshold = weaknessThreshold +// config.blackBox = false } fun repairAction(call: RestCallAction) { @@ -157,7 +162,12 @@ class AIModelsCheck : IntegrationTestRestBase() { val metrics = aiGlobalClassifier.estimateMetrics(action.endpoint) - if (!(metrics.accuracy > 0.5 && metrics.f1Score400 > 0.5)) { + val deltaW = config.aIResponseClassifierWeaknessThreshold + if(metrics.precision400 < deltaW + || metrics.sensitivity400 < deltaW + || metrics.specificity < deltaW + || metrics.npv < deltaW) { + println("The classifier is weak for $endPoint") val result = ExtraTools.executeRestCallAction(action, baseUrlOfSut) @@ -208,11 +218,11 @@ class AIModelsCheck : IntegrationTestRestBase() { val overAllMetrics = aiGlobalClassifier.estimateOverallMetrics() - println("Overall Accuracy: ${overAllMetrics.accuracy}") - println("Overall Precision400: ${overAllMetrics.precision400}") - println("Overall Recall400: ${overAllMetrics.sensitivity400}") - println("Overall F1Score400: ${overAllMetrics.f1Score400}") - println("Overall MCC: ${overAllMetrics.mcc}") + println("Overall Accuracy: ${overAllMetrics.accuracy}") + println("Overall Precision400: ${overAllMetrics.precision400}") + println("Overall Sensitivity400: ${overAllMetrics.sensitivity400}") + println("Overall Specificity: ${overAllMetrics.specificity}") + println("Overall NPV: ${overAllMetrics.npv}") if (saveReport) { saveReports() diff --git a/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/aiclassification/AIModelsCheckWFDEM.kt b/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/aiclassification/AIModelsCheckWFDEM.kt index 4599ec513d..da5a9ac727 100644 --- a/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/aiclassification/AIModelsCheckWFDEM.kt +++ b/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/aiclassification/AIModelsCheckWFDEM.kt @@ -13,16 +13,17 @@ class AIModelsCheckWFDEM : RestTestBase() { } } - val modelName = "KDE" // Choose "GAUSSIAN", "GLM", "KDE", "KNN", "NN", etc. + val modelName = "GAUSSIAN" // Choose "GAUSSIAN", "GLM", "KDE", "KNN", "NN", etc. val encoderType = "RAW" // Choose "RAW" or "NORMAL" - val decisionMaking = "PROBABILITY" // Choose "PROBABILITY" or "THRESHOLD" + val decisionMaking = "THRESHOLD" // Choose "PROBABILITY" or "THRESHOLD" val warmUpRep = 10 val maxAttemptRepair = 100 // i.e., the classifier has 10 times the chances to pick an action with non-400 response val baseUrlOfSut = "http://localhost:8080" // val swaggerUrl = "http://localhost:8080/v2/api-docs" - val swaggerUrl = "http://localhost:8080/api/v3/openapi.json" -// val swaggerUrl ="../WFD_Dataset/openapi-swagger/youtube-mock.yaml" +// val swaggerUrl = "http://localhost:8080/api/v3/openapi.json" +// val swaggerUrl ="../dataset/openapi-swagger/youtube-mock.yaml" + val swaggerUrl ="../dataset/openapi-swagger/catwatch.json" fun runTest() { @@ -33,7 +34,7 @@ class AIModelsCheckWFDEM : RestTestBase() { // Add black-box Swagger parameters args.add("--blackBox") - args.add("true") + args.add("false") args.add("--ratePerMinute") args.add("50000") diff --git a/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/aiclassification/AIMoldelsCheckWFD.kt b/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/aiclassification/AIMoldelsCheckWFD.kt index 272012499b..83adcbc4f6 100644 --- a/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/aiclassification/AIMoldelsCheckWFD.kt +++ b/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/aiclassification/AIMoldelsCheckWFD.kt @@ -43,11 +43,13 @@ class AIModelsCheckWFD : IntegrationTestRestBase() { } } - val modelName = "KDE" // Choose "GAUSSIAN", "GLM", "KDE", "KNN", "NN", etc. + val modelName = "GAUSSIAN" // Choose "GAUSSIAN", "GLM", "KDE", "KNN", "NN", etc. val encoderType = "RAW" // Choose "RAW" or "NORMAL" val decisionMaking = "THRESHOLD" // Choose "PROBABILITY" or "THRESHOLD" val warmUpRep = 10 val maxAttemptRepair = 100 // i.e., the classifier has 10 times the chances to pick an action with non-400 response + val repairThreshold = 0.5 + val weaknessThreshold = 0.6 val runIterations = 1000 val saveReport = false @@ -56,9 +58,13 @@ class AIModelsCheckWFD : IntegrationTestRestBase() { val baseUrlOfSut = "http://localhost:8080" // val swaggerUrl = "http://localhost:8080/v2/api-docs" // val swaggerUrl = "http://localhost:8080/api/v3/openapi.json" - val swaggerUrl ="../WFD_Dataset/openapi-swagger/youtube-mock.yaml" -// val swaggerUrl ="../WFD_Dataset/openapi-swagger/languagetool.json" -// val swaggerUrl = "../WFD_Dataset/openapi-swagger/rest-ncs.json" + +// val swaggerUrl ="../dataset/openapi-swagger/youtube-mock.yaml" + val swaggerUrl ="../dataset/openapi-swagger/catwatch.json" +// val swaggerUrl ="../dataset/openapi-swagger/blogapi.json" +// val swaggerUrl ="../dataset/openapi-swagger/languagetool.json" +// val swaggerUrl = "../dataset/openapi-swagger/rest-ncs.json" +// val swaggerUrl = "../dataset/openapi-swagger/cwa-verification.json" @Inject lateinit var randomness: Randomness @@ -76,6 +82,7 @@ class AIModelsCheckWFD : IntegrationTestRestBase() { injector = init(args) } + fun initializeTest() { recreateInjectorForBlackBox(listOf("--aiModelForResponseClassification", "$modelName")) } @@ -91,6 +98,9 @@ class AIModelsCheckWFD : IntegrationTestRestBase() { config.aiClassifierRepairActivation = EMConfig.AIClassificationRepairActivation.valueOf(decisionMaking) config.aiResponseClassifierWarmup = warmUpRep config.maxRepairAttemptsInResponseClassification = maxAttemptRepair + config.classificationRepairThreshold = repairThreshold + config.aIResponseClassifierWeaknessThreshold = weaknessThreshold + config.blackBox = false } @Inject @@ -138,6 +148,15 @@ class AIModelsCheckWFD : IntegrationTestRestBase() { .joinToString(", ") { ng -> "${ng.gene.name}:${ng.gene::class.simpleName ?: "Unknown"}" }) + println( + "Parameter paths and encoded values are: " + + encoder.getAllParamsPathsAndEncodedValues() + .entries + .joinToString(", ") { (name, value) -> + "$name:$value" + } + ) + if (encoder.areAllGenesUnSupported()) { println("Skipping classification for $endPoint as all its genes are unsupported.") continue @@ -148,8 +167,13 @@ class AIModelsCheckWFD : IntegrationTestRestBase() { println("Input vector size: ${inputVector.size}") // Warm-up - val innerModel = aiGlobalClassifier.viewInnerModels() - println("innerModel is ${innerModel.javaClass.simpleName ?: "Unknown"}") + val innerModels = aiGlobalClassifier.viewInnerModels() + + val innerModel = innerModels.firstOrNull() + ?: throw IllegalStateException("No inner models found") + + println("innerModel is ${innerModel.javaClass.simpleName}") + val endpointModel = when(innerModel) { is Gaussian400Classifier -> innerModel.getModel(endPoint) is GLM400Classifier -> innerModel.getModel(endPoint) @@ -166,7 +190,11 @@ class AIModelsCheckWFD : IntegrationTestRestBase() { val metrics = aiGlobalClassifier.estimateMetrics(action.endpoint) //Execute the action if the classifier is still weak - if(!(metrics.accuracy > 0.5 && metrics.f1Score400 > 0.0 && metrics.mcc > 0.0)){ + val deltaW = config.aIResponseClassifierWeaknessThreshold + if(metrics.precision400 < deltaW + || metrics.sensitivity400 < deltaW + || metrics.specificity < deltaW + || metrics.npv < deltaW) { println("The classifier is weak for $endPoint") val result = ExtraTools.executeRestCallAction(action, "$baseUrlOfSut") @@ -244,11 +272,11 @@ class AIModelsCheckWFD : IntegrationTestRestBase() { } val overAllMetrics = aiGlobalClassifier.estimateOverallMetrics() - println("Overall Accuracy: ${overAllMetrics.accuracy}") - println("Overall Precision400: ${overAllMetrics.precision400}") - println("Overall Recall400: ${overAllMetrics.sensitivity400}") - println("Overall F1Score400: ${overAllMetrics.f1Score400}") - println("Overall MCC: ${overAllMetrics.mcc}") + println("Overall Accuracy: ${overAllMetrics.accuracy}") + println("Overall Precision400: ${overAllMetrics.precision400}") + println("Overall Sensitivity400: ${overAllMetrics.sensitivity400}") + println("Overall Specificity: ${overAllMetrics.specificity}") + println("Overall NPV: ${overAllMetrics.npv}") // Save the final result as a .txt file if (saveReport){ diff --git a/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/aiclassification/ExtraTools.kt b/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/aiclassification/ExtraTools.kt index b441cfa2f8..b327c47ec2 100644 --- a/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/aiclassification/ExtraTools.kt +++ b/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/aiclassification/ExtraTools.kt @@ -58,12 +58,12 @@ object ExtraTools { | Actual 400 | TP=${metrics.truePositive400.toString().padEnd(10)}| FN=${metrics.falseNegative400.toString().padEnd(11)}| | Actual¬400 | FP=${metrics.falsePositive400.toString().padEnd(10)}| TN=${metrics.trueNegative400.toString().padEnd(11)}| +-------------------------------------------+ - Window Total : ${metrics.windowTotal} - Accuracy : ${"%.4f".format(metrics.estimateMetrics().accuracy)} - Precision400 : ${"%.4f".format(metrics.estimateMetrics().precision400)} - Recall400 : ${"%.4f".format(metrics.estimateMetrics().sensitivity400)} - F1Score400 : ${"%.4f".format(metrics.estimateMetrics().f1Score400)} - MCC400 : ${"%.4f".format(metrics.estimateMetrics().mcc)} + Window Total : ${metrics.windowTotal} + Accuracy : ${"%.4f".format(metrics.estimateMetrics().accuracy)} + Precision400 : ${"%.4f".format(metrics.estimateMetrics().precision400)} + Sensitivity400 : ${"%.4f".format(metrics.estimateMetrics().sensitivity400)} + Specificity : ${"%.4f".format(metrics.estimateMetrics().specificity)} + NPV : ${"%.4f".format(metrics.estimateMetrics().npv)} """.trimIndent() ) } @@ -107,12 +107,12 @@ object ExtraTools { sb.appendLine("| Actual¬400 | FP=${it.falsePositive400.toString().padEnd(10)}| TN=${it.trueNegative400.toString().padEnd(11)}|") sb.appendLine("+-------------------------------------------+") sb.appendLine() - sb.appendLine("Window Total : ${it.windowTotal}") - sb.appendLine("Accuracy : ${"%.4f".format(it.estimateMetrics().accuracy)}") - sb.appendLine("Precision400 : ${"%.4f".format(it.estimateMetrics().precision400)}") - sb.appendLine("Recall400 : ${"%.4f".format(it.estimateMetrics().sensitivity400)}") - sb.appendLine("F1Score400 : ${"%.4f".format(it.estimateMetrics().f1Score400)}") - sb.appendLine("MCC400 : ${"%.4f".format(it.estimateMetrics().mcc)}") + sb.appendLine("Window Total : ${it.windowTotal}") + sb.appendLine("Accuracy : ${"%.4f".format(it.estimateMetrics().accuracy)}") + sb.appendLine("Precision400 : ${"%.4f".format(it.estimateMetrics().precision400)}") + sb.appendLine("Sensitivity400 : ${"%.4f".format(it.estimateMetrics().sensitivity400)}") + sb.appendLine("Specificity : ${"%.4f".format(it.estimateMetrics().specificity)}") + sb.appendLine("NPV : ${"%.4f".format(it.estimateMetrics().npv)}") sb.appendLine() sb.appendLine("=============================================") sb.appendLine() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/AbstractProbabilistic400Classifier.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/AbstractProbabilistic400Classifier.kt index 8fac703fff..0a31fa2dfd 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/AbstractProbabilistic400Classifier.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/AbstractProbabilistic400Classifier.kt @@ -38,6 +38,7 @@ abstract class AbstractProbabilistic400Classifier( return } + // create an endPoint model if it does not exist val m = models.getOrPut(endpoint) { val encoder = InputEncoderUtilWrapper(input, encoderType = encoderType) @@ -46,20 +47,26 @@ abstract class AbstractProbabilistic400Classifier( return@getOrPut null } - val listGenes = encoder.endPointToGeneList().map { it.gene.getLeafGene() } + val initialParamPaths = encoder.getAllParamsPathsAndEncodedValues().keys.toList() + createEndpointModel( - endpoint, warmup, - listGenes.size, + endpoint, + warmup, + initialParamPaths, + initialParamPaths.size, encoderType, metricType, - randomness) + randomness + ) } + if (m == null) { unsupportedEndpoints.add(endpoint) return } + // update the endpoint model and initialize if needed m.updateModel(input, output) } @@ -99,6 +106,7 @@ abstract class AbstractProbabilistic400Classifier( protected abstract fun createEndpointModel( endpoint: Endpoint, warmup: Int, + modelKeys: List, dimension: Int, encoderType: EMConfig.EncoderType, metricType: EMConfig.AIClassificationMetrics, diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/AbstractProbabilistic400EndpointModel.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/AbstractProbabilistic400EndpointModel.kt index 8d6fc4375e..dbf1972f6a 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/AbstractProbabilistic400EndpointModel.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/AbstractProbabilistic400EndpointModel.kt @@ -9,51 +9,94 @@ import org.evomaster.core.problem.rest.data.Endpoint import org.evomaster.core.problem.rest.data.RestCallAction import org.evomaster.core.problem.rest.data.RestCallResult import org.evomaster.core.search.service.Randomness +import org.slf4j.LoggerFactory /** * Base class for all probabilistic classifiers working at an endpoint (400 vs. not 400). * - * Provides: - * - Common properties (endpoint, warmup, encoderType, randomness, dimension, initialized flag) - * - Shared methods for initialization checks and accuracy estimation + * This class provides: + * - Common endpoint-specific state (endpoint, warmup, modelKeys, dimension, encoderType, randomness, initialized flag) + * - A fixed feature-space definition via `modelKeys` + * - Shared initialization logic + * - Common performance tracking and metric estimation + * + * The classifier assumes a fixed input representation. Each feature is uniquely identified + * (by e.g., a parameter path) and stored in `modelKeys`, such that: + * + * modelKeys[i] <-> inputVector[i] + * + * where `inputVector` is the encoded representation of a `RestCallAction` produced by [InputEncoderUtilWrapper]. + * The ordering of `modelKeys` is important and must remain stable + * to provide a fixed-length `inputVector` throughout the model's lifetime. + * This guarantees that the same parameter + * is always encoded into the same feature dimension. */ abstract class AbstractProbabilistic400EndpointModel( val endpoint: Endpoint, var warmup: Int, + /** + * Ordered list of model keys in correspondence with the encoded features i.e., + * + * modelKeys[i] <-> inputVector[i] + */ + var modelKeys: List? = null, + /** + * Represents the dimension of the feature space (i.e., the length of the input vector) + */ var dimension: Int? = null, val encoderType: EMConfig.EncoderType, val metricType: EMConfig.AIClassificationMetrics, val randomness: Randomness ) : AIModel { - protected var initialized: Boolean = false - companion object { + + private val log = LoggerFactory.getLogger(AbstractProbabilistic400EndpointModel::class.java) + const val NOT_400 = 200 } + protected var initialized: Boolean = false + /** Create a metric tracker.*/ val modelMetrics: ModelMetrics = createModelMetrics(metricType) - /** Ensure endpoint matches this model */ + /** Ensure the endpoint matches this model */ protected fun verifyEndpoint(inputEndpoint: Endpoint) { if (inputEndpoint != endpoint) { throw IllegalArgumentException("Input endpoint $inputEndpoint does not match model endpoint $endpoint") } } - /** Initialize dimension once, validate consistency */ - open fun initializeIfNeeded(inputVector: List) { - if (dimension == null) { - require(inputVector.isNotEmpty()) { "Input vector cannot be empty" } - require(warmup > 0) { "Warmup must be positive" } - dimension = inputVector.size - } else { - require(inputVector.size == dimension) { - "Expected input vector of size $dimension but got ${inputVector.size}" - } + + /** + * Initialize the model's feature space (including the dimension and modelKeys) if it has not been initialized yet. + * The dimension is the number of parameters in the input vector. + * The modelKeys are unique identifiers of the parameters defined based on + * the unique path of each parameter (including all its parents). + */ + open fun initializeIfNeeded(input: RestCallAction) { + + if (initialized) { + // already initialized. nothing to do + return } + + val encoder = InputEncoderUtilWrapper(input, encoderType = encoderType) + val allParamsPathsAndEncodedValues = encoder.getAllParamsPathsAndEncodedValues() + + val paramPaths = allParamsPathsAndEncodedValues.keys.toList() + + // It is sufficient to check the parameter paths. The encoder guarantees + // that every parameter path has a corresponding non-null encoded value. + require(paramPaths.isNotEmpty()) { "Parameter paths are empty" } + require(warmup > 0) { "Warmup must be positive" } + + dimension = paramPaths.size + modelKeys = paramPaths + initialized = true + } /** @@ -73,6 +116,60 @@ abstract class AbstractProbabilistic400EndpointModel( } + /** + * Encodes the input parameters using the model's defined keys and returns the corresponding encoded values. + * This method ensures that all expected keys (modelKeys) are present in the encoded input. + * + * @param input The input action containing required data for parameter encoding. + * @return A list of encoded values corresponding to the model's keys. + */ + protected fun encodeUsingModelKeys( + input: RestCallAction + ): List { + + val encoder = InputEncoderUtilWrapper( + input, + encoderType = encoderType + ) + + val keysAndValues = encoder.getAllParamsPathsAndEncodedValues() + + // TODO: initializedKeys are based on the model keys that are fixed. + // In fact we ignore new hidden genes added during the search which are not available in the schema. + // A more principled solution would be to support dynamic feature spaces, + // where the classifier can adapt its dimension and feature mapping as + // new parameter structures are discovered during the search. + val initializedKeys = requireNotNull(modelKeys) { + "Model keys have not been initialized" + } + + return initializedKeys.map { key -> + + val value = keysAndValues[key] + + /** + * The encoder never returns null values. + * Therefore, a null means that the encoder provided a new key which is not identical to the model key. + * Hypothetically, this only happens if the parameter path (representing the key) + * changes during the search due to adding an intermediate gene (e.g., the path for + * the parameter b changes from 'GET:/a/b' to 'GET:/a/foo/b'). + * Thus, the encoder returns a different key for the parameter from what is expected. + * In such cases we consider the encoded value as neutral (i.e., 0.0) to avoid errors. + */ + if (value == null) { + log.warn("Missing key while encoding endpoint: {}", endpoint) + log.warn("Model keys: {}", initializedKeys) + log.warn("Current keys: {}", keysAndValues.keys) + log.warn("Missing key: {}", key) + + 0.0 + } else { + value + } + + } + } + /** Default metrics estimates */ override fun estimateMetrics(endpoint: Endpoint): ModelEvaluation { verifyEndpoint(endpoint) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/InputEncoderUtilWrapper.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/InputEncoderUtilWrapper.kt index 72693bd197..ede4265ec5 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/InputEncoderUtilWrapper.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/InputEncoderUtilWrapper.kt @@ -53,8 +53,16 @@ class InputEncoderUtilWrapper( DateTimeGene::class ) + /** + * Represents a mapping between a parameter (with its name and path) + * and its associated gene object. + * @property paramName The name of the parameter. + * @property paramPath A unique identifier for the parameter, representing its hierarchical path (including all its parents). + * @property gene The gene corresponding to the parameter. + */ data class ParamAndGene( val paramName: String, + val paramPath: String, val gene: Gene ) @@ -67,6 +75,35 @@ class InputEncoderUtilWrapper( fun areAllGenesUnSupported(): Boolean = endPointToGeneList().all { !isSupported(it.gene.getLeafGene()) } + /** + * Builds a string representing the gene name and all its parents. + * This string is used as a unique identifier for the gene in the AI models. + */ + private fun genePath(g: Gene): String { + + val names = mutableListOf() + + var current: Gene? = g + + while (current != null) { + names.add(current.name) + current = current.parent as? Gene + } + + val path = names.reversed() + + return if (path.size > 1) + path.dropLast(1).joinToString("/") //ignore the last name, which is the repetition of gene itself as its own parent + else + path.joinToString("/") + + } + + /** + * Recursively expands the input gene into a list of its leaf genes. + * If the input gene is of type [ObjectGene], it will traverse its fixed fields + * and additional fields to expand and collect all nested leaf genes. + */ private fun expandGene(g: Gene): List { val gene = g.getLeafGene() @@ -83,6 +120,23 @@ class InputEncoderUtilWrapper( return listOf(gene) } + + /** + * Associate all parameters' paths with their corresponding encoded numerical values of their parameter. + * Note that each endpoint may have multiple parameters, but each parameter has a unique path including all its parents. + */ + fun getAllParamsPathsAndEncodedValues(): Map { + + val paramPaths = endPointToGeneList().map { it.paramPath } + val encodedValues = encode() + + return paramPaths.zip(encodedValues).toMap() + } + + /** + * Converts the endpoint parameters into a list of `ParamAndGene` objects, + * where each entry represents a parameter, its associated gene, and all the gene's parents. + */ fun endPointToGeneList(): List { val paramAndGenes = mutableListOf() @@ -96,7 +150,13 @@ class InputEncoderUtilWrapper( val g = p.primaryGene() val expanded = expandGene(g) expanded.forEach { subGene -> - paramAndGenes.add(ParamAndGene(subGene.name, subGene)) + paramAndGenes.add( + ParamAndGene( + paramName = subGene.name, + paramPath = genePath(subGene), + gene = subGene + ) + ) } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/gaussian/Gaussian400Classifier.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/gaussian/Gaussian400Classifier.kt index 7bc78345a9..b3161fb7c1 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/gaussian/Gaussian400Classifier.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/gaussian/Gaussian400Classifier.kt @@ -17,6 +17,7 @@ class Gaussian400Classifier( override fun createEndpointModel( endpoint: Endpoint, warmup: Int, + modelKeys: List, dimension: Int, encoderType: EMConfig.EncoderType, metricType: EMConfig.AIClassificationMetrics, @@ -25,9 +26,11 @@ class Gaussian400Classifier( return Gaussian400EndpointModel( endpoint, warmup, + modelKeys, dimension, encoderType, metricType, - randomness) + randomness + ) } -} +} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/gaussian/Gaussian400EndpointModel.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/gaussian/Gaussian400EndpointModel.kt index 9fb7562e61..f39e05e009 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/gaussian/Gaussian400EndpointModel.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/gaussian/Gaussian400EndpointModel.kt @@ -2,7 +2,6 @@ package org.evomaster.core.problem.rest.classifier.probabilistic.gaussian import org.evomaster.core.EMConfig import org.evomaster.core.problem.rest.classifier.AIResponseClassification -import org.evomaster.core.problem.rest.classifier.probabilistic.InputEncoderUtilWrapper import org.evomaster.core.problem.rest.classifier.probabilistic.AbstractProbabilistic400EndpointModel import org.evomaster.core.problem.rest.data.Endpoint import org.evomaster.core.problem.rest.data.RestCallAction @@ -35,12 +34,13 @@ import kotlin.math.ln class Gaussian400EndpointModel ( endpoint: Endpoint, warmup: Int, + modelKeys: List? = null, dimension: Int? = null, encoderType: EMConfig.EncoderType, metricType: EMConfig.AIClassificationMetrics, randomness: Randomness ): AbstractProbabilistic400EndpointModel( - endpoint, warmup, dimension, encoderType, metricType, randomness) { + endpoint, warmup, modelKeys, dimension, encoderType, metricType, randomness) { var density400: Density? = null private set @@ -51,8 +51,9 @@ class Gaussian400EndpointModel ( /** Must be called once to initialize the model properties * Initialize dimension and weights if needed */ - override fun initializeIfNeeded(inputVector: List) { - super.initializeIfNeeded(inputVector) + override fun initializeIfNeeded(input: RestCallAction) { + super.initializeIfNeeded(input) + if(density400 == null) { density400 = Density(dimension!!) } @@ -69,10 +70,7 @@ class Gaussian400EndpointModel ( return AIResponseClassification() } - val encoder = InputEncoderUtilWrapper(input, encoderType = encoderType) - val inputVector = encoder.encode() - - initializeIfNeeded(inputVector) + initializeIfNeeded(input) if (modelMetrics.totalSentRequests < warmup) { // Return equal probabilities during warmup @@ -84,6 +82,8 @@ class Gaussian400EndpointModel ( ) } + val inputVector = encodeUsingModelKeys(input) + if (inputVector.size != dimension) { throw IllegalArgumentException("Expected input vector of size ${this.dimension} but got ${inputVector.size}") } @@ -128,10 +128,9 @@ class Gaussian400EndpointModel ( return } - val encoder = InputEncoderUtilWrapper(input, encoderType = encoderType) - val inputVector = encoder.encode() + initializeIfNeeded(input) - initializeIfNeeded(inputVector) + val inputVector = encodeUsingModelKeys(input) if (inputVector.size != this.dimension) { throw IllegalArgumentException("Expected input vector of size ${this.dimension} but got ${inputVector.size}") diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/glm/GLM400Classifier.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/glm/GLM400Classifier.kt index 1e25e9e50f..8249cc2822 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/glm/GLM400Classifier.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/glm/GLM400Classifier.kt @@ -18,6 +18,7 @@ class GLM400Classifier( override fun createEndpointModel( endpoint: Endpoint, warmup: Int, + modelKeys: List, dimension: Int, encoderType: EMConfig.EncoderType, metricType: EMConfig.AIClassificationMetrics, @@ -26,6 +27,7 @@ class GLM400Classifier( return GLM400EndpointModel( endpoint, warmup, + modelKeys, dimension, encoderType, metricType, diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/glm/GLM400EndpointModel.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/glm/GLM400EndpointModel.kt index b1f0f3a4e9..c6e6357341 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/glm/GLM400EndpointModel.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/glm/GLM400EndpointModel.kt @@ -2,7 +2,6 @@ package org.evomaster.core.problem.rest.classifier.probabilistic.glm import org.evomaster.core.EMConfig import org.evomaster.core.problem.rest.classifier.AIResponseClassification -import org.evomaster.core.problem.rest.classifier.probabilistic.InputEncoderUtilWrapper import org.evomaster.core.problem.rest.classifier.probabilistic.AbstractProbabilistic400EndpointModel import org.evomaster.core.problem.rest.data.Endpoint import org.evomaster.core.problem.rest.data.RestCallAction @@ -20,20 +19,21 @@ import kotlin.math.exp class GLM400EndpointModel( endpoint: Endpoint, warmup: Int, + modelKeys: List? = null, dimension: Int? = null, encoderType: EMConfig.EncoderType, metricType: EMConfig.AIClassificationMetrics, private val learningRate: Double = 0.01, randomness: Randomness ) : AbstractProbabilistic400EndpointModel( - endpoint, warmup, dimension, encoderType, metricType, randomness) { + endpoint, warmup, modelKeys, dimension, encoderType, metricType, randomness){ private var weights: MutableList? = null private var bias: Double = 0.0 /** Initialize dimension and weights if needed */ - override fun initializeIfNeeded(inputVector: List) { - super.initializeIfNeeded(inputVector) + override fun initializeIfNeeded(input: RestCallAction) { + super.initializeIfNeeded(input) if (weights == null) { weights = MutableList(dimension!!) { 0.0 } } @@ -49,10 +49,9 @@ class GLM400EndpointModel( return AIResponseClassification() } - val encoder = InputEncoderUtilWrapper(input, encoderType = encoderType) - val inputVector = encoder.encode() + initializeIfNeeded(input) - initializeIfNeeded(inputVector) + val inputVector = encodeUsingModelKeys(input) if (modelMetrics.totalSentRequests < warmup) { // Return equal probabilities during warmup @@ -96,10 +95,9 @@ class GLM400EndpointModel( return } - val encoder = InputEncoderUtilWrapper(input, encoderType = encoderType) - val inputVector = encoder.encode() + initializeIfNeeded(input) - initializeIfNeeded(inputVector) + val inputVector = encodeUsingModelKeys(input) if (inputVector.size != this.dimension) { throw IllegalArgumentException("Expected input vector of size ${this.dimension} but got ${inputVector.size}") diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/kde/KDE400Classifier.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/kde/KDE400Classifier.kt index eb147abe6e..f67e6aa9e5 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/kde/KDE400Classifier.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/kde/KDE400Classifier.kt @@ -17,6 +17,7 @@ class KDE400Classifier( override fun createEndpointModel( endpoint: Endpoint, warmup: Int, + modelKeys: List, dimension: Int, encoderType: EMConfig.EncoderType, metricType: EMConfig.AIClassificationMetrics, @@ -25,6 +26,7 @@ class KDE400Classifier( return KDE400EndpointModel( endpoint, warmup, + modelKeys, dimension, encoderType, metricType, diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/kde/KDE400EndpointModel.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/kde/KDE400EndpointModel.kt index a39acc7cd3..f99df2fc32 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/kde/KDE400EndpointModel.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/kde/KDE400EndpointModel.kt @@ -2,7 +2,6 @@ package org.evomaster.core.problem.rest.classifier.probabilistic.kde import org.evomaster.core.EMConfig import org.evomaster.core.problem.rest.classifier.AIResponseClassification -import org.evomaster.core.problem.rest.classifier.probabilistic.InputEncoderUtilWrapper import org.evomaster.core.problem.rest.classifier.probabilistic.AbstractProbabilistic400EndpointModel import org.evomaster.core.problem.rest.data.Endpoint import org.evomaster.core.problem.rest.data.RestCallAction @@ -25,21 +24,22 @@ import kotlin.math.sqrt class KDE400EndpointModel ( endpoint: Endpoint, warmup: Int, + modelKeys: List? = null, dimension: Int? = null, encoderType: EMConfig.EncoderType, metricType: EMConfig.AIClassificationMetrics, private val maxStoredSamples: Int = 1000, // Values more than 1000 make the classification very time-consuming randomness: Randomness ): AbstractProbabilistic400EndpointModel( - endpoint, warmup, dimension, encoderType, metricType, randomness) { + endpoint, warmup, modelKeys, dimension, encoderType, metricType, randomness) { private var density400: KDE? = null private var densityNot400: KDE? = null /** Must be called once to initialize the model properties */ - override fun initializeIfNeeded(inputVector: List) { - super.initializeIfNeeded(inputVector) + override fun initializeIfNeeded(input: RestCallAction) { + super.initializeIfNeeded(input) if(density400 == null) { density400 = KDE(dimension!!, maxStoredSamples, randomness) } @@ -57,10 +57,9 @@ class KDE400EndpointModel ( return AIResponseClassification() } - val encoder = InputEncoderUtilWrapper(input, encoderType = encoderType) - val inputVector = encoder.encode() + initializeIfNeeded(input) - initializeIfNeeded(inputVector) + val inputVector = encodeUsingModelKeys(input) if (modelMetrics.totalSentRequests < warmup) { // Return equal probabilities during warmup @@ -116,10 +115,10 @@ class KDE400EndpointModel ( return } - val encoder = InputEncoderUtilWrapper(input, encoderType = encoderType) - val inputVector = encoder.encode() - initializeIfNeeded(inputVector) + initializeIfNeeded(input) + + val inputVector = encodeUsingModelKeys(input) if (inputVector.size != this.dimension) { throw IllegalArgumentException("Expected input vector of size ${this.dimension} but got ${inputVector.size}") diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/knn/KNN400Classifier.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/knn/KNN400Classifier.kt index 3e2d9ae9d6..31ac730683 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/knn/KNN400Classifier.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/knn/KNN400Classifier.kt @@ -4,6 +4,7 @@ import org.evomaster.core.EMConfig import org.evomaster.core.problem.rest.data.Endpoint import org.evomaster.core.search.service.Randomness import org.evomaster.core.problem.rest.classifier.probabilistic.AbstractProbabilistic400Classifier +import kotlin.String class KNN400Classifier( @@ -19,6 +20,7 @@ class KNN400Classifier( override fun createEndpointModel( endpoint: Endpoint, warmup: Int, + modelKeys: List, dimension: Int, encoderType: EMConfig.EncoderType, metricType: EMConfig.AIClassificationMetrics, @@ -27,6 +29,7 @@ class KNN400Classifier( return KNN400EndpointModel( endpoint, warmup, + modelKeys, dimension, encoderType, metricType, diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/knn/KNN400EndpointModel.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/knn/KNN400EndpointModel.kt index 9e584f00ae..857bfc7ce5 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/knn/KNN400EndpointModel.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/knn/KNN400EndpointModel.kt @@ -2,7 +2,6 @@ package org.evomaster.core.problem.rest.classifier.probabilistic.knn import org.evomaster.core.EMConfig import org.evomaster.core.problem.rest.classifier.AIResponseClassification -import org.evomaster.core.problem.rest.classifier.probabilistic.InputEncoderUtilWrapper import org.evomaster.core.problem.rest.classifier.probabilistic.AbstractProbabilistic400EndpointModel import org.evomaster.core.problem.rest.data.Endpoint import org.evomaster.core.problem.rest.data.RestCallAction @@ -20,6 +19,7 @@ import kotlin.math.sqrt class KNN400EndpointModel ( endpoint: Endpoint, warmup: Int, + modelKeys: List? = null, dimension: Int? = null, encoderType: EMConfig.EncoderType, metricType: EMConfig.AIClassificationMetrics, @@ -27,13 +27,13 @@ class KNN400EndpointModel ( private val maxStoredSamples: Int = 1000, // Values more than 1000 make the classification very time-consuming randomness: Randomness ): AbstractProbabilistic400EndpointModel( - endpoint, warmup, dimension, encoderType, metricType, randomness) { + endpoint, warmup, modelKeys, dimension, encoderType, metricType, randomness) { /** * Stores the training samples for this endpoint model. * Each element is a pair of: * - List: the encoded feature vector - * - Int : the corresponding status code (i.e., HTTP response) + * - Int: the corresponding status code (i.e., HTTP response) */ val samples = mutableListOf, Int>>() @@ -53,10 +53,9 @@ class KNN400EndpointModel ( return AIResponseClassification() } - val encoder = InputEncoderUtilWrapper(input, encoderType = encoderType) - val inputVector = encoder.encode() + initializeIfNeeded(input) - initializeIfNeeded(inputVector) + val inputVector = encodeUsingModelKeys(input) if (modelMetrics.totalSentRequests < warmup) { // Return equal probabilities during warmup @@ -96,10 +95,9 @@ class KNN400EndpointModel ( return } - val encoder = InputEncoderUtilWrapper(input, encoderType = encoderType) - val inputVector = encoder.encode() + initializeIfNeeded(input) - initializeIfNeeded(inputVector) + val inputVector = encodeUsingModelKeys(input) if (inputVector.size != this.dimension) { throw IllegalArgumentException("Expected input vector of size ${this.dimension} but got ${inputVector.size}") diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/nn/NN400Classifier.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/nn/NN400Classifier.kt index a893bf30e1..5e69b1c9c2 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/nn/NN400Classifier.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/nn/NN400Classifier.kt @@ -18,6 +18,7 @@ class NN400Classifier( override fun createEndpointModel( endpoint: Endpoint, warmup: Int, + modelKeys: List, dimension: Int, encoderType: EMConfig.EncoderType, metricType: EMConfig.AIClassificationMetrics, @@ -26,6 +27,7 @@ class NN400Classifier( return NN400EndpointModel( endpoint, warmup, + modelKeys, dimension, encoderType, metricType, diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/nn/NN400EndpointModel.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/nn/NN400EndpointModel.kt index d87f5915af..aee190aa9f 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/nn/NN400EndpointModel.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/classifier/probabilistic/nn/NN400EndpointModel.kt @@ -2,7 +2,6 @@ package org.evomaster.core.problem.rest.classifier.probabilistic.nn import org.evomaster.core.EMConfig import org.evomaster.core.problem.rest.classifier.AIResponseClassification -import org.evomaster.core.problem.rest.classifier.probabilistic.InputEncoderUtilWrapper import org.evomaster.core.problem.rest.classifier.probabilistic.AbstractProbabilistic400EndpointModel import org.evomaster.core.problem.rest.data.Endpoint import org.evomaster.core.problem.rest.data.RestCallAction @@ -20,13 +19,14 @@ import kotlin.math.exp class NN400EndpointModel( endpoint: Endpoint, warmup: Int, + modelKeys: List? = null, dimension: Int? = null, encoderType: EMConfig.EncoderType, metricType: EMConfig.AIClassificationMetrics, private val learningRate: Double = 0.01, randomness: Randomness ) : AbstractProbabilistic400EndpointModel( - endpoint, warmup, dimension, encoderType, metricType, randomness) { + endpoint, warmup, modelKeys, dimension, encoderType, metricType, randomness) { // Initialize weights with default values to prevent null @@ -37,24 +37,14 @@ class NN400EndpointModel( /** Must be called once to initialize the model properties */ - override fun initializeIfNeeded(inputVector: List) { - if (!initialized || dimension == null) { - require(inputVector.isNotEmpty()) { "Input vector cannot be empty" } - require(warmup > 0) { "Warmup must be positive" } - dimension = inputVector.size - - // Initialize with proper dimensions - weightsInputHidden = Array(dimension!!) { - DoubleArray(hiddenSize) { randomness.nextDouble(-0.1, 0.1) } - } - weightsHiddenOutput = Array(hiddenSize) { - DoubleArray(outputSize) { randomness.nextDouble(-0.1, 0.1) } - } - initialized = true - } else { - require(inputVector.size == dimension) { - "Expected input vector of size $dimension but got ${inputVector.size}" - } + override fun initializeIfNeeded(input: RestCallAction) { + super.initializeIfNeeded(input) + // Initialize with proper dimensions + weightsInputHidden = Array(dimension!!) { + DoubleArray(hiddenSize) { randomness.nextDouble(-0.1, 0.1) } + } + weightsHiddenOutput = Array(hiddenSize) { + DoubleArray(outputSize) { randomness.nextDouble(-0.1, 0.1) } } } @@ -69,19 +59,9 @@ class NN400EndpointModel( return AIResponseClassification() } - val encoder = InputEncoderUtilWrapper(input, encoderType = encoderType) - val inputVector = encoder.encode() + initializeIfNeeded(input) - if (encoder.areAllGenesUnSupported()) { - // skip classification/training if unsupported - return AIResponseClassification( - probabilities = mapOf( - NOT_400 to 0.5, - 400 to 0.5) - ) - } - - initializeIfNeeded(inputVector) + val inputVector = encodeUsingModelKeys(input) if (modelMetrics.totalSentRequests < warmup) { // Return equal probabilities during warmup @@ -113,18 +93,9 @@ class NN400EndpointModel( return } - val encoder = InputEncoderUtilWrapper(input, encoderType = encoderType) - val inputVector = encoder.encode() - - if (encoder.areAllGenesUnSupported() || inputVector.isEmpty()) { - // Skip training if unsupported or empty - val predictedStatusCode = if(randomness.nextBoolean()) 400 else NOT_400 - modelMetrics.updatePerformance(predictedStatusCode,output.getStatusCode()?:-1) - modelMetrics.updatePerformance(predictedStatusCode, output.getStatusCode()?:-1) - return - } + initializeIfNeeded(input) - initializeIfNeeded(inputVector) + val inputVector = encodeUsingModelKeys(input) if (inputVector.size != this.dimension) { throw IllegalArgumentException("Expected input vector of size ${this.dimension} but got ${inputVector.size}")