diff --git a/obp-api/src/main/scala/code/api/constant/constant.scala b/obp-api/src/main/scala/code/api/constant/constant.scala index 2cb79f2af5..80cfdaa6ba 100644 --- a/obp-api/src/main/scala/code/api/constant/constant.scala +++ b/obp-api/src/main/scala/code/api/constant/constant.scala @@ -320,7 +320,12 @@ object Constant extends MdcLoggable { final val CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL: Int = APIUtil.getPropsValue(s"createLocalisedResourceDocJson.cache.ttl.seconds", "3600").toInt final val GET_DYNAMIC_RESOURCE_DOCS_TTL: Int = APIUtil.getPropsValue(s"dynamicResourceDocsObp.cache.ttl.seconds", "3600").toInt final val GET_STATIC_RESOURCE_DOCS_TTL: Int = APIUtil.getPropsValue(s"staticResourceDocsObp.cache.ttl.seconds", "3600").toInt - final val SHOW_USED_CONNECTOR_METHODS: Boolean = APIUtil.getPropsAsBoolValue(s"show_used_connector_methods", false) + // def, not final val: DynamicUtil.Validation.validateDependency (dynamic-code dependency + // checking) needs this to react to a props change without a restart -- e.g. test-time + // setPropsValues overrides. A final val here would freeze at whatever value was true the + // moment this object was first touched (typically during server boot, well before any test + // scenario runs), and no later prop override could ever reach it. + def SHOW_USED_CONNECTOR_METHODS: Boolean = APIUtil.getPropsAsBoolValue(s"show_used_connector_methods", false) // Rate Limiting Cache Prefixes (with global namespace and versioning) // Both call_counter and rl_active are versioned for consistent cache invalidation diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala index beaad62688..97b5c34459 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala @@ -77,7 +77,7 @@ trait EndpointGroup { * @param successResponseBody successResponseBody from the post json body,it is JValue here. * @param methodBody it is url-encoded string for the api level code. */ -case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String) { +case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String, programmingLang: String = "Scala") { val decodedMethodBody = URLDecoder.decode(methodBody, "UTF-8") val requestBody: Product = exampleRequestBody match { //this case means, we accept the empty string "" from json post body, we need to map it to None. @@ -87,7 +87,24 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo } val successResponse: Product = toCaseObject(successResponseBody) - private val partialFunction: Http4sEndpointIO = { + private val partialFunction: Http4sEndpointIO = programmingLang match { + case "java" | "Java" => + DynamicUtil.createJavaHttp4sEndpoint(decodedMethodBody) match { + case Full(func) => func + case Failure(msg: String, exception: Box[Throwable], _) => + throw exception.getOrElse(new RuntimeException(msg)) + case _ => throw new RuntimeException("compiled code return nothing") + } + case _ /* "Scala" | "scala" | "" | null, default */ => + scalaPartialFunction + } + + // Unchanged Scala-template compile path, factored out so the `partialFunction` match above stays + // readable. Only evaluated for Scala-language docs (the default) — Java-language docs never + // touch this, so example/response-body JValues that don't fit the Scala case-class generator + // (irrelevant for Java, since it doesn't use RequestRootJsonClass/ResponseRootJsonClass) are a + // non-issue there. + private def scalaPartialFunction: Http4sEndpointIO = { //If the requestBody is PrimaryDataBody, return None. otherwise, return the exampleRequestBody:Option[JValue] // In side OBP resourceDoc, requestBody and successResponse must be Product type， @@ -157,8 +174,21 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo * this will check all the dynamic scala code dependencies at compile time. * *Search for the usage, you can see how to use it in OBP code. + * + * Scala-only: for the Scala language, `this.partialFunction` IS the compiled user code, so + * validating its bytecode directly is correct. For Java, `this.partialFunction` is instead + * OBP's own Http4sEndpointIO wrapper (built by DynamicUtil.createJavaHttp4sEndpoint) around the + * real compiled Java class -- its bytecode legitimately calls internal OBP helpers + * (DynamicUtil.javaValueToJValue/logger, CustomJsonFormats.formats, JsonAliases.compactRender) + * that were never meant to be dependency-whitelisted, since they are framework glue, not + * user-supplied code. createJavaHttp4sEndpoint already validates the real compiled Java class + * internally (see its own doc comment) before ever returning that wrapper, so re-validating the + * wrapper here is both redundant and wrong -- it would reject every Java doc unconditionally. */ - def validateDependency() = Validation.validateDependency(this.partialFunction) + def validateDependency() = programmingLang match { + case "java" | "Java" => () + case _ => Validation.validateDependency(this.partialFunction) + } /** * This is used to check the security permission at the run time. diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala index 8e6350494b..a20ce39ea1 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala @@ -21,6 +21,22 @@ object DynamicResourceDocsEndpointGroup extends EndpointGroup with code.util.Hel try { Some(toResourceDoc(dynamicDoc)) } catch { + // Validation.validateDependency / createJavaHttp4sEndpoint's own rejection path both throw + // this specifically for a dependency-whitelist miss -- distinct from a genuine compile + // failure, and reachable here (not just at create/update time) because CompiledObjects' + // validation runs fresh on every construction and dynamic_code_compile_validate_dependencies + // can be tightened after a doc was already registered. Logging it as a "deprecated Lift + // contract" problem sends whoever reads this log to re-author a body that is not the + // problem, instead of at the whitelist they (or someone else) just edited. + case e: code.api.JsonResponseException => + val reason = e.jsonResponse match { + case APIUtil.JsonResponseExtractor(msg, _) => msg + case _ => Option(e.getMessage).getOrElse("") + } + logger.error(s"[DynamicResourceDocsEndpointGroup] skipping dynamic resource doc '${dynamicDoc.requestVerb} ${dynamicDoc.requestUrl}' " + + s"(id=${dynamicDoc.dynamicResourceDocId.getOrElse("")}, programming_lang=${dynamicDoc.programmingLang}): rejected by dependency " + + s"validation (dynamic_code_compile_validate_dependencies). $reason") + None case e: Throwable => logger.error(s"[DynamicResourceDocsEndpointGroup] skipping dynamic resource doc '${dynamicDoc.requestVerb} ${dynamicDoc.requestUrl}' " + s"(id=${dynamicDoc.dynamicResourceDocId.getOrElse("")}): its methodBody could not be compiled under the native http4s contract. " + @@ -49,7 +65,7 @@ object DynamicResourceDocsEndpointGroup extends EndpointGroup with code.util.Hel * */ private val toResourceDoc: JsonDynamicResourceDoc => ResourceDoc = { dynamicDoc => - val compiledObjects = CompiledObjects(dynamicDoc.exampleRequestBody, dynamicDoc.successResponseBody, dynamicDoc.methodBody) + val compiledObjects = CompiledObjects(dynamicDoc.exampleRequestBody, dynamicDoc.successResponseBody, dynamicDoc.methodBody, dynamicDoc.programmingLang) ResourceDoc( // partialFunction is a no-op stub — the runtime dispatch uses the native handler in // dynamicHttp4sFunction (the compiled artifact is OBPEndpointIO, not the Lift OBPEndpoint). diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index b9b110654c..7a317068ac 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -4396,8 +4396,10 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ * * than the return value may be (getUserAndSessionContextFuture, ***,***),(map,***,***), (getOrElse,***,***) ...... */ - def getDependentMethods(className: String, methodName:String, signature: String): List[(String, String, String)] = { - if (SHOW_USED_CONNECTOR_METHODS) { + // force bypasses the SHOW_USED_CONNECTOR_METHODS gate below -- see + // DynamicUtil.getDynamicCodeDependentMethods' doc comment for why security validation needs this. + def getDependentMethods(className: String, methodName:String, signature: String, force: Boolean = false): List[(String, String, String)] = { + if (SHOW_USED_CONNECTOR_METHODS || force) { val methods = ListBuffer[(String, String, String)]() //NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. //eg: className == code.api.UKOpenBanking.v3_1_0.APIMethods_AccountAccessApi$$anonfun$createAccountAccessConsents$lzycompute$1 diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 862bf609c8..c38de119c4 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -43,7 +43,20 @@ object DynamicUtil extends MdcLoggable{ } val toolBox: ToolBox[universe.type] = runtimeMirror(getClass.getClassLoader).mkToolBox() + // Neither this nor memoJavaCompiledScript below ever evicts, so each distinct ClassLoader (and + // therefore each distinct compiled Java method_body -- java-scriptengine hands createJavaHttp4sEndpoint + // a fresh MemoryClassLoader per compile) is retained for the life of the process, along with its + // ClassPool. This is the same unbounded-but-trusted-operator-only tradeoff dynamicCompileResult + // below already makes for the Scala compile cache, predating the Java path: registering a dynamic + // resource doc is gated behind canCreateDynamicResourceDoc / canCreateBankLevelDynamicResourceDoc, + // not open to arbitrary callers, and a served endpoint's ClassLoader must stay reachable for as + // long as that endpoint keeps serving requests -- an eviction policy here would need to be + // reference-counted against currently-registered docs to avoid reclaiming a live one, which is a + // larger change than this cache's existing (pre-Java) design accounted for. private val memoClassPool = new Memo[ClassLoader, ClassPool] + // Caches only the compiled artifact (deterministic given the source string), never the + // validation outcome built on top of it -- see createJavaHttp4sEndpoint's doc comment. + private val memoJavaCompiledScript = new Memo[String, Box[ch.obermuhlner.scriptengine.java.JavaCompiledScript]] private def getClassPool(classLoader: ClassLoader) = memoClassPool.memoize(classLoader){ val cp = ClassPool.getDefault @@ -167,29 +180,71 @@ object DynamicUtil extends MdcLoggable{ } /** - * NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. + * NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. * @param clazz * @param predicate + * @param force bypasses the SHOW_USED_CONNECTOR_METHODS gate below. SHOW_USED_CONNECTOR_METHODS + * exists to opt in to an unrelated, expensive introspection/reporting feature (which + * connector methods a static endpoint touches) — it was never meant to gate SECURITY + * validation, which reuses this same bytecode scan. Without `force`, a deployment + * that sets dynamic_code_compile_validate_enable=true (the documented, security- + * relevant prop) but leaves the unrelated show_used_connector_methods at its default + * false would silently get an always-empty dependency list here — every dynamic-code + * call looks "allowed" no matter what it does, because there is nothing to check + * against the whitelist. Validation.validateDependency passes force=true so it is + * controlled solely by dynamic_code_compile_validate_enable, matching what an + * operator following that prop's own documentation would expect. * @return */ - def getDynamicCodeDependentMethods(clazz: Class[_], predicate: String => Boolean = _ => true): List[(String, String, String)] = - if (SHOW_USED_CONNECTOR_METHODS) { + def getDynamicCodeDependentMethods(clazz: Class[_], predicate: String => Boolean = _ => true, force: Boolean = false): List[(String, String, String)] = + if (SHOW_USED_CONNECTOR_METHODS || force) { val className = clazz.getTypeName val listBuffer = new ListBuffer[(String, String, String)]() val classPool = getClassPool(clazz.getClassLoader) - //NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. + //NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. val ctClass = classPool.get(className) + + // A same-class (or same-generated-unit, for the Scala nested-closure case below) call is not + // itself a dependency to police -- recurse into what the TARGET method calls instead of + // flagging the call itself as forbidden, all the way down until a genuinely foreign + // dependency is reached. This is required for Java: every Java dynamic resource doc + // implements Supplier> (the documented convention), and the + // compiler always erases that generic Supplier.get() to a synthetic bridge method + // `Object get()` whose body is just `return this.get();` -- an ordinary same-class + // invokevirtual call to the real, properly-typed get(). A single level of unrolling only + // fixes that one hop: any Java body that factors logic into its own private helper methods + // (an entirely normal thing to do) reintroduces the exact same false rejection one level + // deeper, since the un-recursed helper's own callees would otherwise be appended as raw + // (thisClass, method) tuples and then rejected as calls to an unwhitelistable random-UUID + // class. `visited` guards against a call cycle -- direct or mutual recursion between + // same-class private methods (e.g. a fibonacci/factorial helper) is entirely normal Java and + // would otherwise recurse forever. On hitting a cycle this contributes nothing further (Nil), + // not a leaf: the recursive call is still a same-class call, not a foreign dependency, and + // whatever it in turn depends on is already being expanded by the in-progress call further up + // this same path -- returning it as a leaf here would flag the method's own name + // (unwhitelistable, like any other randomly-named dynamic class) as a forbidden dependency, + // exactly the bug this whole function exists to avoid. + def expand(typeName: String, methodName: String, signature: String, visited: Set[(String, String, String)]): List[(String, String, String)] = { + val key = (typeName, methodName, signature) + val sameUnit = typeName == className || + (className.startsWith(typeName) && methodName.startsWith(clazz.getPackage.getName + "$")) + if (!sameUnit) { + List(key) + } else if (visited.contains(key)) { + Nil + } else { + APIUtil.getDependentMethods(typeName, methodName, signature, force).flatMap { case (t, m, s) => + expand(t, m, s, visited + key) + } + } + } + for { method <- ctClass.getDeclaredMethods.toList if predicate(method.getName) - ternary @ (typeName, methodName, signature) <- APIUtil.getDependentMethods(className, method.getName, method.getSignature) + (typeName, methodName, signature) <- APIUtil.getDependentMethods(className, method.getName, method.getSignature, force) } yield { - // if method is also dynamic compile code, extract it's dependent method - if(className.startsWith(typeName) && methodName.startsWith(clazz.getPackage.getName+ "$")) { - listBuffer.appendAll(APIUtil.getDependentMethods(typeName, methodName, signature)) - } else { - listBuffer.append(ternary) - } + listBuffer.appendAll(expand(typeName, methodName, signature, Set.empty)) } listBuffer.distinct.toList @@ -360,6 +415,22 @@ object DynamicUtil extends MdcLoggable{ object Validation { + // def, not val, throughout this object: these must react to a props change (e.g. test-time + // setPropsValues) without a restart, not freeze at whatever the props held the moment + // Validation was first touched (typically by whichever dynamic-code test happens to run + // first in a shared test JVM). This costs nothing extra in production -- the only expensive + // step, DynamicUtil.compileScalaCodeUnchecked, is already memoized by the exact source + // string, so re-evaluating these on every call is a cache hit unless the underlying props + // value actually changed. + // + // This makes allowedRuntimePermissions itself always current, but NOT everything downstream + // of it: Sandbox.sandbox(bankId) below separately caches the whole Sandbox it builds, keyed + // only by bankId -- so a bankId whose sandbox was already built keeps that snapshot of + // allowedRuntimePermissions until the process restarts, same staleness this def change fixed + // for validateDependency. Left as-is here because it's moot in practice: SecurityManager + // enforcement is already a no-op on this JVM (JEP 486, JDK 24+; see Sandbox's own comment), + // so neither the stale nor the fresh permission list is actually enforced. + /** * Turn the `dynamic_code_compile_validate_dependencies` props value into the Scala source * that, once compiled, yields the whitelist. @@ -380,10 +451,10 @@ object DynamicUtil extends MdcLoggable{ dependenciesString.replaceFirst("\\[", "Map[String, String](").dropRight(1) + ").mapValues(v => StringUtils.split(v, ',').map(_.trim).toSet).toMap" - val dynamicCodeSandboxPermissions = APIUtil.getPropsValue("dynamic_code_sandbox_permissions", "[]").trim - val scalaCodePermissioins = "List[java.security.Permission]"+dynamicCodeSandboxPermissions.replaceFirst("\\[","(").dropRight(1)+")" - val permissions:Box[List[java.security.Permission]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodePermissioins) - + def dynamicCodeSandboxPermissions = APIUtil.getPropsValue("dynamic_code_sandbox_permissions", "[]").trim + def scalaCodePermissioins = "List[java.security.Permission]"+dynamicCodeSandboxPermissions.replaceFirst("\\[","(").dropRight(1)+")" + def permissions:Box[List[java.security.Permission]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodePermissioins) + // all Permissions put at here // Here is the Java Permission document, please extend these permissions carefully. // https://docs.oracle.com/javase/8/docs/technotes/guides/security/spec/security-spec.doc3.html#17001 @@ -402,11 +473,11 @@ object DynamicUtil extends MdcLoggable{ // new RuntimePermission("accessDeclaredMembers"), // new RuntimePermission("getClassLoader"), // ) - val allowedRuntimePermissions = permissions.openOrThrowException("Can not compile the props `dynamic_code_sandbox_permissions` to permissions") + def allowedRuntimePermissions = permissions.openOrThrowException("Can not compile the props `dynamic_code_sandbox_permissions` to permissions") - val dependenciesString = APIUtil.getPropsValue("dynamic_code_compile_validate_dependencies", "[]").trim - val scalaCodeDependencies = dependenciesScalaCode(dependenciesString) - val dependenciesBox: Box[Map[String, Set[String]]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodeDependencies) + def dependenciesString = APIUtil.getPropsValue("dynamic_code_compile_validate_dependencies", "[]").trim + def scalaCodeDependencies = dependenciesScalaCode(dependenciesString) + def dependenciesBox: Box[Map[String, Set[String]]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodeDependencies) /** * Compilation OBP Dependencies Guard, only checked the OBP methods, not scala/Java libraies(are checked during the runtime.). @@ -437,7 +508,7 @@ object DynamicUtil extends MdcLoggable{ // PractiseEndpoint.getClass.getTypeName + "*" -> "*", // // ).mapValues(v => StringUtils.split(v, ',').map(_.trim).toSet) - val allowedCompilationMethods: Map[String, Set[String]] = dependenciesBox.openOrThrowException("Can not compile the props `dynamic_code_compile_validate_dependencies` to Map") + def allowedCompilationMethods: Map[String, Set[String]] = dependenciesBox.openOrThrowException("Can not compile the props `dynamic_code_compile_validate_dependencies` to Map") //Do not touch this Set, try to use the `allowedPermissions` and `allowedMethods` to control the sandbox val restrictedTypes = Set( @@ -454,6 +525,11 @@ object DynamicUtil extends MdcLoggable{ * Here only validate the restricted types(isObpClass + val restrictedTypes), not all scala/java types. */ private def validateDependency(dependentMethods: List[(String, String, String)]) = { + // Bound once per call, not re-derived per dependency tuple: allowedCompilationMethods is a + // def (see the "def, not val" comment above) so it observes a live props change, but it + // recompiles the whitelist source on every access -- reading it twice per element inside + // the `collect` guard below would mean up to 2N re-derivations for N dependency tuples. + val allowedCompilationMethods = this.allowedCompilationMethods val notAllowedDependentMethods = dependentMethods collect { case (typeName, method, _) if isRestrictedType(typeName) && @@ -476,7 +552,9 @@ object DynamicUtil extends MdcLoggable{ def validateDependency(obj: AnyRef): Unit = { if(APIUtil.getPropsAsBoolValue("dynamic_code_compile_validate_enable",false)){ - val dependentMethods: List[(String, String, String)] = DynamicUtil.getDynamicCodeDependentMethods(obj.getClass) + // force=true: this check must not also require the unrelated show_used_connector_methods + // prop -- see getDynamicCodeDependentMethods' doc comment for why. + val dependentMethods: List[(String, String, String)] = DynamicUtil.getDynamicCodeDependentMethods(obj.getClass, force = true) validateDependency(dependentMethods) } else{ // If false, nothing to do here. ; @@ -559,4 +637,196 @@ object DynamicUtil extends MdcLoggable{ } } } + + /** + * Converts a plain value returned by a compiled Java `method_body` into a JValue, for endpoints + * where json4s' `Extraction.decompose` cannot help: it works by Scala-case-class/collection + * reflection, so a `java.util.Map`/`java.util.List` returned from Java decomposes to `{}`/`[]` + * (its entries are invisible to Scala reflection) rather than throwing — a silent data-loss bug, + * not a compile or runtime error, so it only surfaces as an empty response body. Recurses through + * the Java collection types directly; anything else (including a Scala case class constructed + * from Java, as ConnectorMethod's Java example does) falls back to Extraction.decompose. + */ + private def javaValueToJValue(value: Any): JValue = { + import scala.jdk.CollectionConverters._ + value match { + case null => JNull + case jv: JValue => jv + case m: java.util.Map[_, _] => + JObject(m.asScala.toList.map { case (k, v) => (String.valueOf(k), javaValueToJValue(v)) }) + case l: java.util.List[_] => + JArray(l.asScala.toList.map(javaValueToJValue)) + case s: String => JString(s) + case b: java.lang.Boolean => JBool(b) + case i: java.lang.Integer => JInt(BigInt(i.intValue())) + case l: java.lang.Long => JInt(BigInt(l.longValue())) + case d: java.lang.Double => JDouble(d.doubleValue()) + case f: java.lang.Float => JDouble(f.doubleValue()) + case bd: java.math.BigDecimal => JDecimal(BigDecimal(bd)) + case other => Extraction.decompose(other)(CustomJsonFormats.formats) + } + } + + /** + * Compiles a Java `method_body` for a DynamicResourceDoc endpoint into a native + * `Http4sEndpointIO` (`PartialFunction[Request[IO], CallContext => IO[Response[IO]]]`), the same + * type the Scala template compiles to in DynamicEndpoints.CompiledObjects. + * + * Reuses the same JSR-223 "java" engine (backed by a real javax.tools.JavaCompiler via + * ch.obermuhlner:java-scriptengine — see createJavaFunction above) and the same + * package-uniquification trick, but — unlike createJavaFunction, whose DynamicFunction shape is + * specific to the ConnectorMethod feature — wraps the compiled function in a hand-written + * Http4sEndpointIO here in Scala. The Java method_body never has to construct cats.effect.IO, + * org.http4s.Response, or a Scala PartialFunction: it only ever returns a plain Java object + * (Map/List/String/number/boolean/etc.), which this adapter serializes via javaValueToJValue + * above (NOT Extraction.decompose directly — see that method's doc comment for why). + * + * Java-side convention (identical to the existing ConnectorMethod convention): the pasted class + * implements java.util.function.Supplier>. The + * compiled function is invoked with: + * args(0) = the raw request body (String, or null if the request had none) + * args(1) = path params (java.util.Map) + * args(2) = the CallContext (present whenever this endpoint is actually being served) + * mirroring createJavaFunction's own `func(args ++ cc)` call (line above): appending an + * Option[CallContext] via `++` appends its *contents* (0 or 1 raw CallContext), not the Option + * wrapper itself, so Java reads args[2] directly as a CallContext, no unwrapping needed. + * + * Unlike createJavaFunction, this validates the actual compiled Java class (not just its Scala + * wrapper) against `dynamic_code_compile_validate_dependencies`/`dynamic_code_compile_validate_enable`. + * CompiledObjects.validateDependency() (called by the ResourceDoc-creation flow) only ever sees + * `this.partialFunction` — the hand-written Http4sEndpointIO below — whose own bytecode just + * calls `java.util.function.Function.apply`, a non-restricted type; it can't see what the pasted + * Java class does inside apply(Object[]). Worse, `func` itself (the Function returned by the + * pasted class's get()) is commonly a method reference (`this::apply`), which the JVM + * materialises as a synthetic lambda class whose bytecode is just a delegating call — validating + * `func.getClass` would be equally blind. So we go through the JSR-223 Compilable API directly + * (JavaScriptEngine implements it) instead of plain eval(), to get the real top-level compiled + * class/instance (JavaCompiledScript.getCompiledClass/getCompiledInstance) and validate that + * before the function is ever returned or invoked. + */ + def createJavaHttp4sEndpoint(methodBody: String): Box[code.api.util.APIUtil.Http4sEndpointIO] = + if (!dynamicCodeExecutionEnabled) Failure(ErrorMessages.DynamicCodeExecutionDisabled) + else { + import cats.effect.IO + import code.api.util.APIUtil.Http4sEndpointIO + import com.openbankproject.commons.ExecutionContext.Implicits.global + import com.openbankproject.commons.util.JsonAliases.compactRender + import org.http4s.headers.`Content-Type` + import org.http4s.dsl.io._ + import org.http4s.{MediaType, Request, Response} + + import scala.jdk.CollectionConverters._ + + // Only the compile step is memoized — deterministic given the same source string, and the + // one genuinely expensive part (a real javax.tools.JavaCompiler invocation). Dependency + // validation below is NOT memoized: it depends on mutable external config + // (dynamic_code_compile_validate_enable/_dependencies), which can change between two + // createJavaHttp4sEndpoint calls for the identical source string — e.g. a doc compiled once + // while validation was off, then a later create/update call resubmitting the exact same + // method_body after validation was turned on and the whitelist tightened. An earlier version + // of this function memoized the validated *result* (Box[Http4sEndpointIO]) as a single unit, + // so that second call silently reused the first call's unvalidated success — bypassing the + // now-stricter policy for any resubmitted source. Re-running validation on every call costs + // little: it is Javassist bytecode inspection plus a Map lookup, not another compile. + val compiledScriptBox: Box[ch.obermuhlner.scriptengine.java.JavaCompiledScript] = + memoJavaCompiledScript.memoize("java-http4s-endpoint:" + methodBody) { + // Real compile happens here (javax.tools.JavaCompiler via the JSR-223 "java" engine) — + // any Java syntax/type error surfaces as an exception, caught by this `Box tryo` and + // turned into a Failure. + Box tryo { + val packageExp = UUID.randomUUID().toString.replaceAll("^|-", "_") + val packageMatcher = Pattern.compile("""(?m)^\s*package\s+\S+?\s*;""").matcher(methodBody) + + val javaCode = s"""package code.api.util.dynamic.${packageExp}; + |${packageMatcher.replaceFirst("")} + |""".stripMargin + + val compiledScript = javaEngine.asInstanceOf[javax.script.Compilable].compile(javaCode) + .asInstanceOf[ch.obermuhlner.scriptengine.java.JavaCompiledScript] + + // getDynamicCodeDependentMethods loads a class's bytecode via Javassist's + // LoaderClassPath, which reads it through classLoader.getResourceAsStream(...). The + // compiler's ch.obermuhlner.scriptengine.java.MemoryClassLoader only overrides + // loadClass() — it never exposes the compiled bytes as a classpath resource — so that + // lookup silently fails (javassist.NotFoundException) and validation would see zero + // dependent methods no matter what the Java code actually calls. Read the bytes + // directly from the classloader's private byte map (reflection is unavoidable here: + // java-scriptengine exposes no public accessor) and hand them to Javassist explicitly + // via ByteArrayClassPath, so the real method bodies — including any restricted OBP + // call — are visible to validation. Done here, inside the compile memoization, so it + // runs exactly once per distinct source: ClassPool.appendClassPath has no dedup of its + // own, so doing this on every createJavaHttp4sEndpoint call (as an earlier version of + // this function did, on every resourceDocs-list rebuild for the process's lifetime) + // grew that ClassPool's classpath chain without bound. + val compiledClass = compiledScript.getCompiledClass + val classBytesField = compiledClass.getClassLoader.getClass.getDeclaredField("mapClassBytes") + classBytesField.setAccessible(true) + val classBytes = classBytesField.get(compiledClass.getClassLoader) + .asInstanceOf[java.util.Map[String, Array[Byte]]].get(compiledClass.getName) + // Fail loudly and specifically here rather than handing Javassist a null byte array -- + // that would only surface later, inside ByteArrayClassPath/ClassPool, as an opaque NPE + // with no indication that the cause was this reflective read (e.g. a java-scriptengine + // upgrade that changes mapClassBytes' keying from binary name to internal name, or that + // stops using that field name at all). + if (classBytes == null) { + throw new IllegalStateException( + s"createJavaHttp4sEndpoint: MemoryClassLoader.mapClassBytes has no entry for " + + s"${compiledClass.getName} -- java-scriptengine's internal layout may have changed") + } + getClassPool(compiledClass.getClassLoader) + .appendClassPath(new javassist.ByteArrayClassPath(compiledClass.getName, classBytes)) + + compiledScript + } + } + + // Deliberately outside compiledScriptBox's `Box tryo` AND outside the memoization above: a + // rejection here throws JsonResponseException, which must propagate UNCAUGHT (mirroring the + // Scala path's CompiledObjects.validateDependency(), also never wrapped in tryo) so + // compileDynamicResourceDoc's `case e: JsonResponseException => throw e` sees it intact. + // JsonResponseException never sets a Throwable message (getMessage == null); Box.tryo would + // catch it into Failure(null, Full(theException), Empty), and DynamicEndpoints.scala's + // `case Failure(msg: String, ...)` pattern silently fails to match a null msg — falling + // through to "compiled code return nothing" and discarding the real rejection reason. `.map` + // does not swallow exceptions the way `Box tryo` does, so this stays uncaught here. + compiledScriptBox.map { compiledScript => + // Validate the real compiled Supplier class before it's ever invoked — see the doc comment + // above for why this must run against getCompiledInstance, not `func`/`this.partialFunction`, + // and why it must run fresh on every call rather than being cached with the compile result. + Validation.validateDependency(compiledScript.getCompiledInstance) + + val func = compiledScript.eval().asInstanceOf[java.util.function.Function[Array[AnyRef], Any]] + val jsonContentType = `Content-Type`(MediaType.application.json) + + new Http4sEndpointIO { + override def isDefinedAt(req: Request[IO]): Boolean = true + + override def apply(req: Request[IO]): CallContext => IO[Response[IO]] = { cc => + val pathParams: java.util.Map[String, String] = cc.resourceDocument + .map(_.getPathParams(req.uri.path.segments.toList.map(_.encoded))) + .getOrElse(Map.empty[String, String]) + .asJava + + val valueIO: IO[Any] = IO.fromFuture(IO { + Future { + val args: Array[AnyRef] = Array(cc.httpBody.orNull, pathParams) + func(args ++ Some(cc)) + } + }) + + valueIO.flatMap { value => + Ok(compactRender(javaValueToJValue(value)), jsonContentType) + }.handleErrorWith { e => + logger.warn(s"createJavaHttp4sEndpoint: Java method_body threw", e) + InternalServerError( + compactRender(Extraction.decompose( + Map("code" -> 500, "message" -> s"OBP-50000: Unknown Error. ${e.getMessage}") + )(CustomJsonFormats.formats)), + jsonContentType + ) + } + } + } + } + } } diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala index 51840cd239..b54ddd7662 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala @@ -9414,12 +9414,19 @@ object Http4s400 { case _ => true } } + // Fail fast with a clean 400 before attempting compilation, rather than surfacing an + // unsupported programming_lang only as a generic DynamicCodeCompileFail. + _ <- code.util.Helper.booleanToFuture( + s"""$DynamicCodeLangNotSupport programming_lang ${body.programmingLang}, currently supported languages: Scala, Java""", + cc = Some(cc)) { + Set("", "scala", "Scala", "java", "Java").contains(body.programmingLang) + } } yield () } private def compileDynamicResourceDoc(body: JsonDynamicResourceDoc, cc: CallContext): Unit = { try { - CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody).validateDependency() + CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang).validateDependency() } catch { case e: JsonResponseException => throw e case e: Exception => diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index a4bb0fe6b6..d3a2d4dc6d 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -4427,12 +4427,23 @@ object Http4s600 { case _ => true } } + // Mirrors Http4s400's validateDynamicResourceDocBody: fail fast on an unsupported + // programming_lang here too, rather than reporting `valid = true` for a language + // create would actually reject with 400 DynamicCodeLangNotSupport (CompiledObjects + // silently falls through to the Scala compile path for any value it doesn't + // recognise as Java, so an unsupported/misspelled language would otherwise still + // "validate" successfully as Scala). + _ <- Helper.booleanToFuture( + s"""$DynamicCodeLangNotSupport programming_lang ${body.programmingLang}, currently supported languages: Scala, Java""", + cc = Some(cc)) { + Set("", "scala", "Scala", "java", "Java").contains(body.programmingLang) + } } yield try { code.api.dynamic.endpoint.helper.CompiledObjects( - body.exampleRequestBody, body.successResponseBody, body.methodBody).validateDependency() + body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang).validateDependency() ValidateDynamicResourceDocSuccessJsonV600( valid = true, - message = "Dynamic Resource Doc method body is valid Scala and uses allowed dependencies.") + message = s"Dynamic Resource Doc method body is valid ${body.programmingLang} and uses allowed dependencies.") } catch { case e: code.api.JsonResponseException => val errorText = e.jsonResponse match { diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala index a96c164350..d42ef429fd 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala @@ -25,6 +25,9 @@ class DynamicResourceDoc extends LongKeyedMapper[DynamicResourceDoc] with IdPK w object Tags extends MappedString(this, 255) object Roles extends MappedString(this, 255) object MethodBody extends MappedText(this) + // Source language of MethodBody: "Scala" (default) or "Java". Mirrors DynamicMessageDoc.Lang / + // ConnectorMethod.programmingLang — same field name/width convention, see DynamicEndpoints. + object Lang extends MappedString(this, 50) // Provenance: who created / last updated this runtime-compiled endpoint, and a SHA-256 of the // (decoded) method body so tampering / drift is detectable. Set server-side from the CallContext // user — never from the request body. createdAt / updatedAt come from the CreatedUpdated trait. @@ -50,7 +53,12 @@ object DynamicResourceDoc extends DynamicResourceDoc with LongKeyedMetaMapper[Dy successResponseBody = Option(dynamicResourceDoc.SuccessResponseBody.get).filter(StringUtils.isNotBlank).map(json.parse), errorResponseBodies = dynamicResourceDoc.ErrorResponseBodies.get, tags = dynamicResourceDoc.Tags.get, - roles = dynamicResourceDoc.Roles.get + roles = dynamicResourceDoc.Roles.get, + // Rows created before the Lang column existed have NULL there, not "Scala" -- a bare + // Lang.get would surface that as an empty/null programming_lang instead of falling back to + // JsonDynamicResourceDoc's own "Scala" default, since an explicit null argument bypasses a + // case class default (that only applies when the argument is omitted entirely). + programmingLang = Option(dynamicResourceDoc.Lang.get).filter(StringUtils.isNotBlank).getOrElse("Scala") ) } diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala index 60e94a243a..aca89c7f76 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala @@ -33,7 +33,12 @@ case class JsonDynamicResourceDoc( successResponseBody: Option[JValue], errorResponseBodies: String, tags: String, - roles: String + roles: String, + // Source language of methodBody: "Scala" (default) or "Java". Mirrors + // JsonConnectorMethod.programmingLang / JsonDynamicMessageDoc.programmingLang. Appended last + // (not inserted alphabetically) so existing named-arg call sites and JSON payloads that predate + // this field keep compiling/deserializing unchanged. + programmingLang: String = "Scala" ) extends JsonFieldReName { def decodedMethodBody: String = URLDecoder.decode(methodBody, "UTF-8") } diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala index 47be5d0442..7b8d25a700 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala @@ -82,6 +82,7 @@ object MappedDynamicResourceDocProvider extends DynamicResourceDocProvider { .Tags(entity.tags) .Roles(entity.roles) .MethodBody(entity.methodBody) + .Lang(entity.programmingLang) // provenance is set here from the authenticated user + computed hash, not from `entity` .CreatedByUserId(createdByUserId.getOrElse(null)) .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) @@ -107,6 +108,7 @@ object MappedDynamicResourceDocProvider extends DynamicResourceDocProvider { .Tags(entity.tags) .Roles(entity.roles) .MethodBody(entity.methodBody) + .Lang(entity.programmingLang) // CreatedByUserId is left untouched; record who last changed the code + refresh the hash .UpdatedByUserId(updatedByUserId.getOrElse(null)) .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) diff --git a/obp-api/src/test/resources/frozen_type_meta_data b/obp-api/src/test/resources/frozen_type_meta_data index a15d8f1900..a49f5f72b3 100644 Binary files a/obp-api/src/test/resources/frozen_type_meta_data and b/obp-api/src/test/resources/frozen_type_meta_data differ diff --git a/obp-api/src/test/resources/frozen_type_meta_data.txt b/obp-api/src/test/resources/frozen_type_meta_data.txt index d7bddd02ce..bf2252ef5b 100644 --- a/obp-api/src/test/resources/frozen_type_meta_data.txt +++ b/obp-api/src/test/resources/frozen_type_meta_data.txt @@ -511,7 +511,6 @@ field code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON.SeverJWK use String field code.api.util.APIUtil.BooleanBody value Boolean field code.api.util.APIUtil.EndpointInfo name String field code.api.util.APIUtil.EndpointInfo version String -field code.api.util.APIUtil.JArrayBody value org.json4s.JArray field code.api.v1_2_1.APIInfoJSON connector String field code.api.v1_2_1.APIInfoJSON git_commit String field code.api.v1_2_1.APIInfoJSON hosted_by code.api.v1_2_1.HostedBy @@ -3145,6 +3144,7 @@ field code.dynamicResourceDoc.JsonDynamicResourceDoc errorResponseBodies String field code.dynamicResourceDoc.JsonDynamicResourceDoc exampleRequestBody Option[org.json4s.JValue] field code.dynamicResourceDoc.JsonDynamicResourceDoc methodBody String field code.dynamicResourceDoc.JsonDynamicResourceDoc partialFunctionName String +field code.dynamicResourceDoc.JsonDynamicResourceDoc programmingLang String field code.dynamicResourceDoc.JsonDynamicResourceDoc requestUrl String field code.dynamicResourceDoc.JsonDynamicResourceDoc requestVerb String field code.dynamicResourceDoc.JsonDynamicResourceDoc roles String diff --git a/obp-api/src/test/scala/code/api/util/DynamicUtilJavaHttp4sEndpointTest.scala b/obp-api/src/test/scala/code/api/util/DynamicUtilJavaHttp4sEndpointTest.scala new file mode 100644 index 0000000000..ab984a317b --- /dev/null +++ b/obp-api/src/test/scala/code/api/util/DynamicUtilJavaHttp4sEndpointTest.scala @@ -0,0 +1,109 @@ +package code.api.util + +import cats.effect.IO +import cats.effect.unsafe.implicits.global +import org.http4s.{Method, Request, Uri} +import org.json4s.native.JsonMethods.parse +import org.scalatest.{FeatureSpec, GivenWhenThen, Matchers} + +/** + * Focused unit test for DynamicUtil.createJavaHttp4sEndpoint, isolated from the full + * register -> role-check -> HTTP-dispatch round trip (see DynamicResourceDocJavaTest for that). + * Exercises the adapter directly: compiled Java Supplier> -> + * Http4sEndpointIO.apply(Request[IO]) -> CallContext => IO[Response[IO]]. + */ +class DynamicUtilJavaHttp4sEndpointTest extends FeatureSpec with Matchers with GivenWhenThen { + + private val echoMethodBody = + """package code.api.util.dynamic; + | + |import code.api.util.CallContext; + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaHttp4sEndpointUnitTest implements Supplier> { + | private Object apply(Object[] args) { + | String rawBody = (String) args[0]; + | @SuppressWarnings("unchecked") + | Map pathParams = (Map) args[1]; + | CallContext cc = (CallContext) args[2]; + | + | Map response = new LinkedHashMap<>(); + | response.put("echoed_body", rawBody); + | response.put("path_param_count", pathParams.size()); + | response.put("correlation_id", cc.correlationId()); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + feature("DynamicUtil.createJavaHttp4sEndpoint compiles a Java method_body into a native Http4sEndpointIO") { + + scenario("the compiled endpoint reads args(0)/args(1)/args(2) and serves 200 JSON") { + Given("a Java method_body compiled via createJavaHttp4sEndpoint") + val endpoint = DynamicUtil.createJavaHttp4sEndpoint(echoMethodBody).openOrThrowException("compilation failed") + + When("the compiled endpoint handles a request carrying a body, no path params, and a CallContext") + val req = Request[IO](method = Method.POST, uri = Uri.unsafeFromString("/test")) + val cc = CallContext(httpBody = Some("""{"hello":"world"}"""), correlationId = "test-correlation-id") + val resp = endpoint.apply(req)(cc).unsafeRunSync() + + Then("the response is 200 and echoes the body, the (empty) path params, and the CallContext's correlationId") + resp.status.code should equal(200) + val bodyString = resp.body.through(fs2.text.utf8.decode).compile.string.unsafeRunSync() + val json = parse(bodyString) + (json \ "echoed_body").values should equal("""{"hello":"world"}""") + (json \ "path_param_count").values should equal(BigInt(0)) + (json \ "correlation_id").values should equal("test-correlation-id") + } + + scenario("a Java compile error is reported as a Box Failure, not a thrown exception") { + Given("a method_body that is not valid Java") + val badMethodBody = "this is not valid java at all" + + When("we try to compile it") + val result = DynamicUtil.createJavaHttp4sEndpoint(badMethodBody) + + Then("compilation fails gracefully") + result.isDefined should equal(false) + } + + scenario("a Java method_body that throws at runtime is recovered as a 500, not an uncaught exception") { + Given("a Java method_body whose apply() throws") + val throwingMethodBody = + """package code.api.util.dynamic; + | + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaHttp4sEndpointThrowingTest implements Supplier> { + | private Object apply(Object[] args) { + | throw new RuntimeException("boom"); + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + val endpoint = DynamicUtil.createJavaHttp4sEndpoint(throwingMethodBody).openOrThrowException("compilation failed") + + When("the compiled endpoint is invoked") + val req = Request[IO](method = Method.POST, uri = Uri.unsafeFromString("/test")) + val resp = endpoint.apply(req)(CallContext()).unsafeRunSync() + + Then("the response is 500 rather than the IO failing") + resp.status.code should equal(500) + val bodyString = resp.body.through(fs2.text.utf8.decode).compile.string.unsafeRunSync() + bodyString should include("boom") + } + } +} diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala new file mode 100644 index 0000000000..72f6cc6934 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala @@ -0,0 +1,174 @@ +package code.api.v4_0_0 + +import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.util.ApiRole +import code.api.util.ErrorMessages.DynamicResourceDocMethodDependency +import code.entitlement.Entitlement +import com.openbankproject.commons.model.ErrorMessage +import org.json4s.native.Serialization.write + +/** + * With dynamic_code_compile_validate_enable=true, a Java method_body that calls an OBP method NOT + * on the dependency whitelist must be rejected -- proving createJavaHttp4sEndpoint validates the + * real compiled Java class (getCompiledInstance), not just its Scala wrapper. + */ +class DynamicResourceDocJavaSecurityValidationTest extends V400ServerSetup { + + private def maliciousMethodBody: String = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaSecurityProbe implements Supplier> { + | private Object apply(Object[] args) { + | // APIUtil.getPropsValue is NOT on dynamic_code_compile_validate_dependencies' + | // whitelist (only errorJsonResponse*/scalaFutureToLaFuture/futureToBoxedResponse are). + | String secret = code.api.util.APIUtil$.MODULE$.getPropsValue("hostname", "none"); + | Map response = new LinkedHashMap<>(); + | response.put("leaked", secret); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + // Mirrors sample.props.template's default dynamic_code_compile_validate_dependencies exactly, + // minus the trailing newlines/line-continuations -- deliberately does NOT list APIUtil.getPropsValue. + private def defaultDependenciesWhitelist: String = + """[NewStyle.function.getClass.getTypeName -> "*", CompiledObjects.getClass.getTypeName -> "sandbox", HttpCode.getClass.getTypeName -> "200", DynamicCompileEndpoint.getClass.getTypeName -> "getPathParams, scalaFutureToBoxedJsonResponse", APIUtil.getClass.getTypeName -> "errorJsonResponse, errorJsonResponse$default$1, errorJsonResponse$default$2, errorJsonResponse$default$3, errorJsonResponse$default$4, scalaFutureToLaFuture, futureToBoxedResponse", ErrorMessages.getClass.getTypeName -> "*", ExecutionContext.Implicits.getClass.getTypeName -> "global", JSONFactory400.getClass.getTypeName -> "createBanksJson", classOf[Sandbox].getTypeName -> "runInSandbox", classOf[CallContext].getTypeName -> "*", classOf[ResourceDoc].getTypeName -> "getPathParams", "scala.reflect.runtime.package$" -> "universe", PractiseEndpoint.getClass.getTypeName + "*" -> "*"]""" + + // Deliberately does NOT set show_used_connector_methods: that prop exists to opt in to an + // unrelated, expensive introspection/reporting feature and was never meant to gate security + // validation, which happens to reuse the same underlying bytecode scan. An operator who reads + // only dynamic_code_compile_validate_enable's own prop documentation and sets just these two + // props (as this method does) must still get real enforcement -- proving that is the point of + // every scenario below. + // + // Block body (not `= setPropsValues(...)`) so .github/scripts/check_test_isolation.py's brace + // scanner sees an opening `{` right after `def enableStrictValidation` and treats this as a + // safe "helper called from scenarios" scope rather than a class-body-level setPropsValues call. + private def enableStrictValidation(): Unit = { + setPropsValues( + "dynamic_code_compile_validate_enable" -> "true", + "dynamic_code_compile_validate_dependencies" -> defaultDependenciesWhitelist + ) + } + + // Every Java method_body implements Supplier> per convention (see + // DynamicUtil.createJavaHttp4sEndpoint's doc comment). javac always erases that generic + // Supplier.get() to a synthetic bridge method `Object get()` whose body just invokevirtual-calls + // the real, properly-typed get() -- an ordinary same-class call regardless of what the body does. + private def benignMethodBody: String = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaSecurityBenignProbe implements Supplier> { + | private Object apply(Object[] args) { + | Map response = new LinkedHashMap<>(); + | response.put("greeting", "hello"); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + private def createRequest = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + + feature("Security validation of Java method_body against dynamic_code_compile_validate_dependencies") { + + // Regression guard: the Supplier.get() generics-erasure bridge method's same-class call to the + // real get() must not itself be treated as a call to a forbidden method. Without this, every + // Java doc -- malicious or not -- was rejected under strict validation, because the compiled + // class lives under the OBP-owned code.* package but its randomly-generated name can never + // appear in a static whitelist. + scenario("Registering a benign Java doc succeeds even with strict validation enabled") { + enableStrictValidation() + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "benignProbeTest", + requestUrl = "/benign_probe_test/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(benignMethodBody, "UTF-8"), + programmingLang = "Java" + ) + val resp = makePostRequest(createRequest, write(doc)) + + Then("the compile succeeds -- the Supplier.get() bridge method's self-call is not a forbidden dependency") + resp.code should equal(201) + } + scenario("Registering a Java doc that calls a non-whitelisted OBP method is rejected with 400") { + enableStrictValidation() + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "securityProbeTest", + requestUrl = "/security_probe_test/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(maliciousMethodBody, "UTF-8"), + programmingLang = "Java" + ) + val resp = makePostRequest(createRequest, write(doc)) + + Then("the compile is rejected with 400 DynamicResourceDocMethodDependency, not accepted") + resp.code should equal(400) + resp.body.extract[ErrorMessage].message should include(DynamicResourceDocMethodDependency) + } + + // Regression guard for the bug createJavaHttp4sEndpoint had before it split compilation from + // validation: memoJavaCompiledScript (formerly memoJavaHttp4sEndpoint) memoized the WHOLE + // Box[Http4sEndpointIO], keyed only by the exact method_body string. A doc compiled once while + // validation was off got a cached Full(...) that a later, identical create call -- made AFTER + // validation was turned on and the whitelist tightened -- would silently reuse, never + // re-running Validation.validateDependency at all. This scenario reproduces exactly that + // sequence: compile the same malicious source once with validation off (succeeds, populates + // the compile cache), then enable strict validation and resubmit the identical source. + scenario("A Java source compiled once while validation was off is still validated on a later create call") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val firstDoc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "cacheBypassProbeTest1", + requestUrl = "/cache_bypass_probe_test_1/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(maliciousMethodBody, "UTF-8"), + programmingLang = "Java" + ) + When("validation is off (the suite default) and we compile the malicious source for the first time") + val firstResp = makePostRequest(createRequest, write(firstDoc)) + firstResp.code should equal(201) + + When("validation is then turned on with the same source resubmitted under a different doc") + enableStrictValidation() + val secondDoc = firstDoc.copy( + partialFunctionName = "cacheBypassProbeTest2", + requestUrl = "/cache_bypass_probe_test_2/MY_USER_ID" + ) + val secondResp = makePostRequest(createRequest, write(secondDoc)) + + Then("the second create is still rejected -- the compile-result cache must not bypass fresh validation") + secondResp.code should equal(400) + secondResp.body.extract[ErrorMessage].message should include(DynamicResourceDocMethodDependency) + } + } +} diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala new file mode 100644 index 0000000000..aa12c85bc8 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala @@ -0,0 +1,141 @@ +package code.api.v4_0_0 + +import org.json4s._ +import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.util.ApiRole +import code.api.util.ErrorMessages.DynamicCodeLangNotSupport +import code.dynamicResourceDoc.JsonDynamicResourceDoc +import code.entitlement.Entitlement +import com.openbankproject.commons.model.ErrorMessage +import com.openbankproject.commons.util.json +import org.json4s.native.JsonMethods.{compact, parse => parseJson, render} +import org.json4s.native.Serialization.write + +/** + * Java-language coverage for the DynamicResourceDoc runtime-compilation mechanism. + * DynamicResourceDocTest.scala covers the (unchanged) Scala-language path end-to-end; these + * scenarios exercise the new `programming_lang = "Java"` dispatch added to + * DynamicEndpoints.CompiledObjects / DynamicUtil.createJavaHttp4sEndpoint, plus the + * backward-compat and unsupported-language guards added alongside it. + */ +class DynamicResourceDocJavaTest extends V400ServerSetup { + + private def createDynamicResourceDocsRequest = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + + // Java-side convention: the pasted class implements Supplier>. + // args(0) = raw request body (String, or null), args(1) = path params (java.util.Map), + // args(2) = the CallContext. See DynamicUtil.createJavaHttp4sEndpoint's doc comment. + private def javaRoleTestMethodBody: String = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaResourceDocRoleTest implements Supplier> { + | private Object apply(Object[] args) { + | String rawBody = (String) args[0]; + | @SuppressWarnings("unchecked") + | Map pathParams = (Map) args[1]; + | String myUserId = pathParams.get("MY_USER_ID"); + | + | Map response = new LinkedHashMap<>(); + | response.put("user_id_from_path", myUserId + "_from_path"); + | response.put("received_body", rawBody); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + feature("Native execution of a runtime-compiled dynamic resource doc with a Java method_body") { + + scenario("Create a role-gated Java-language dynamic resource doc and verify 401 / 403 / 200") { + val dynamicRole = "CanCallJavaPieceCRoleTest" + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + When("We create a Java-language dynamic resource doc gated by that role") + val createReq = createDynamicResourceDocsRequest + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = dynamicRole, + partialFunctionName = "javaPieceCRoleTest", + requestUrl = "/my_java_role_user/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(javaRoleTestMethodBody, "UTF-8"), + programmingLang = "Java" + ) + val createResp = makePostRequest(createReq, write(doc)) + Then("We should get a 201") + createResp.code should equal(201) + createResp.body.extract[JsonDynamicResourceDoc].programmingLang should equal("Java") + + val callUrl = dynamicEndpoint_Request / "dynamic-resource-doc" / "my_java_role_user" / "user-1" + val body = """{"name":"Jhon","age":12,"hobby":["coding"]}""" + + assertRoleGated401Then403Then200(callUrl, body, dynamicRole) { resp200 => + val rendered = json.compactRender(resp200.body) + rendered should include("user-1_from_path") + rendered should include("Jhon") + } + } + + scenario("Reject an unsupported programming_lang before attempting compilation") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + When("We create a dynamic resource doc with an unsupported programming_lang") + val createReq = createDynamicResourceDocsRequest + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + partialFunctionName = "unsupportedLangTest", + requestUrl = "/unsupported_lang_test/MY_USER_ID", + programmingLang = "Python" + ) + val resp = makePostRequest(createReq, write(doc)) + + Then("We should get a 400 DynamicCodeLangNotSupport, not a compile-failure error") + resp.code should equal(400) + resp.body.extract[ErrorMessage].message should include(DynamicCodeLangNotSupport) + } + + scenario("Backward compatibility: a request body with programming_lang entirely omitted still creates a Scala-language doc") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + When("We create a dynamic resource doc from a JSON payload that predates the programming_lang field") + val posted = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "preExistingClientTest", + requestUrl = "/pre_existing_client_test/MY_USER_ID" + ) + // Simulate an old client payload: strip programming_lang out entirely rather than relying on + // Serialization.write, which always emits every case-class field (default or not). + val fullJson = parseJson(write(posted)) + val withoutLang = fullJson.removeField { case (name, _) => name == "programming_lang" } + val requestBodyStr = compact(render(withoutLang)) + requestBodyStr should not include "programming_lang" + + val createReq = createDynamicResourceDocsRequest + val createResp = makePostRequest(createReq, requestBodyStr) + + Then("We should get a 201 and the stored/served doc defaults to the Scala language") + createResp.code should equal(201) + createResp.body.extract[JsonDynamicResourceDoc].programmingLang should equal("Scala") + + Then("calling the endpoint still compiles and serves via the (unchanged) Scala template path") + val callReq = (dynamicEndpoint_Request / "dynamic-resource-doc" / "pre_existing_client_test" / "user-1").POST <@ (user1) + val callResp = makePostRequest(callReq, """{"name":"Jhon","age":12,"hobby":["coding"]}""") + callResp.code should equal(200) + val rendered = json.compactRender(callResp.body) + rendered should include("user-1_from_path") + rendered should include("Jhon") + } + } +} diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala index a482557a18..c7d21954af 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala @@ -29,7 +29,7 @@ import org.json4s._ import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON import code.api.util.APIUtil.OAuth._ import code.api.util.ApiRole._ -import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, DynamicResourceDocAlreadyExists, DynamicResourceDocNotFound, UserHasMissingRoles} +import code.api.util.ErrorMessages.{DynamicResourceDocAlreadyExists, DynamicResourceDocNotFound, UserHasMissingRoles} import code.api.util.ApiRole import code.api.v4_0_0.APIMethods400.Implementations4_0_0 import code.dynamicResourceDoc.JsonDynamicResourceDoc @@ -315,21 +315,9 @@ class DynamicResourceDocTest extends V400ServerSetup { val callUrl = dynamicEndpoint_Request / "dynamic-resource-doc" / "my_role_user" / "user-1" val body = """{"name":"Jhon","age":12,"hobby":["coding"]}""" - Then("calling without authentication returns 401") - val resp401 = makePostRequest(callUrl.POST, body) - resp401.code should equal(401) - resp401.body.extract[ErrorMessage].message should include(AuthenticatedUserIsRequired) - - Then("calling authenticated but without the role returns 403") - val resp403 = makePostRequest(callUrl.POST <@ (user1), body) - resp403.code should equal(403) - resp403.body.extract[ErrorMessage].message should include(UserHasMissingRoles) - - Then("granting the role makes the call succeed (200)") - Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, dynamicRole) - val resp200 = makePostRequest(callUrl.POST <@ (user1), body) - resp200.code should equal(200) - json.compactRender(resp200.body) should include("_from_path") + assertRoleGated401Then403Then200(callUrl, body, dynamicRole) { resp200 => + json.compactRender(resp200.body) should include("_from_path") + } } // Regression guard for DynamicEndpointCodeGenerator.buildTemplate: the template served by @@ -426,6 +414,42 @@ class DynamicResourceDocTest extends V400ServerSetup { storedRow.UpdatedByUserId.get should be(resourceUser1.userId) storedRow.MethodBodyHash.get should be(code.api.util.APIUtil.sha256Hex(URLDecoder.decode(changedMethodBody, "UTF-8"))) } + + // Regression guard: rows created before the Lang column existed have a genuine SQL NULL there + // (Schemifier's ALTER TABLE ADD COLUMN sets no default), not "Scala". An explicit null argument + // bypasses JsonDynamicResourceDoc's own programmingLang="Scala" default -- that default only + // applies when the argument is omitted entirely -- so a bare Lang.get would have surfaced as a + // null/empty programming_lang in the API response instead of falling back to "Scala". + scenario("A dynamic resource doc row predating the programming_lang column still reports \"Scala\"", ApiEndpoint1, ApiEndpoint3, VersionOfApi) { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canGetDynamicResourceDoc.toString) + + When("We create a dynamic resource doc") + val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val posted = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + partialFunctionName = "preDatesLangColumnTest", + requestUrl = "/pre_dates_lang_column_test/MY_USER_ID" + ) + val createResp = makePostRequest(createReq, write(posted)) + createResp.code should equal(201) + val docId = (createResp.body \ "dynamic_resource_doc_id").values.toString + + When("its lang column is forced to a genuine SQL NULL, bypassing the ORM (which always writes \"Scala\")") + import code.dynamicResourceDoc.DynamicResourceDoc + net.liftweb.mapper.DB.runUpdate( + s"UPDATE ${DynamicResourceDoc.dbTableName} SET ${DynamicResourceDoc.Lang.dbColumnName} = NULL " + + s"WHERE ${DynamicResourceDoc.DynamicResourceDocId.dbColumnName} = ?", + List(docId) + ) + + Then("GET still reports programming_lang as \"Scala\", not null or empty") + val getReq = (v4_0_0_Request / "management" / "dynamic-resource-docs" / docId).GET <@ (user1) + val getResp = makeGetRequest(getReq) + getResp.code should equal(200) + getResp.body.extract[JsonDynamicResourceDoc].programmingLang should equal("Scala") + } } } diff --git a/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala b/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala index ba69e99330..e33eee4eb5 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala @@ -21,9 +21,10 @@ import code.metadata.comments.MappedComment import code.metadata.narrative.MappedNarrative import code.metadata.transactionimages.MappedTransactionImage import code.metadata.wheretags.MappedWhereTag +import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, UserHasMissingRoles} import code.setup.{APIResponse, DefaultUsers, ServerSetupWithTestData} import code.transactionattribute.MappedTransactionAttribute -import com.openbankproject.commons.model.{AccountId, AccountRoutingJsonV121, AmountOfMoneyJsonV121, BankId, CreateViewJson, UpdateViewJSON} +import com.openbankproject.commons.model.{AccountId, AccountRoutingJsonV121, AmountOfMoneyJsonV121, BankId, CreateViewJson, ErrorMessage, UpdateViewJSON} import com.openbankproject.commons.util.ApiShortVersions import code.setup.OBPReq import org.json4s.native.Serialization.write @@ -42,6 +43,30 @@ trait V400ServerSetup extends ServerSetupWithTestData with DefaultUsers { def dynamicEndpoint_Request: OBPReq = baseRequest / "obp" / ApiShortVersions.`dynamic-endpoint`.toString def dynamicEntity_Request: OBPReq = baseRequest / "obp" / ApiShortVersions.`dynamic-entity`.toString + /** + * Shared by DynamicResourceDocTest and DynamicResourceDocJavaTest: exercises + * ResourceDoc.authCheckIO's role-gated path against a runtime-compiled dynamic-resource-doc -- + * calling without auth returns 401, authenticated without the role returns 403, and granting the + * role makes the call succeed (200), handed to `assertSuccess` for endpoint-specific checks. + */ + def assertRoleGated401Then403Then200(callUrl: OBPReq, body: String, dynamicRole: String)(assertSuccess: APIResponse => Unit): Unit = { + Then("calling without authentication returns 401") + val resp401 = makePostRequest(callUrl.POST, body) + resp401.code should equal(401) + resp401.body.extract[ErrorMessage].message should include(AuthenticatedUserIsRequired) + + Then("calling authenticated but without the role returns 403") + val resp403 = makePostRequest(callUrl.POST <@ (user1), body) + resp403.code should equal(403) + resp403.body.extract[ErrorMessage].message should include(UserHasMissingRoles) + + Then("granting the role makes the call succeed (200)") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, dynamicRole) + val resp200 = makePostRequest(callUrl.POST <@ (user1), body) + resp200.code should equal(200) + assertSuccess(resp200) + } + def randomBankId : String = { def getBanksInfo : APIResponse = { val request = v4_0_0_Request / "banks" diff --git a/obp-api/src/test/scala/code/api/v6_0_0/ValidateDynamicResourceDocTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/ValidateDynamicResourceDocTest.scala new file mode 100644 index 0000000000..7637619c6b --- /dev/null +++ b/obp-api/src/test/scala/code/api/v6_0_0/ValidateDynamicResourceDocTest.scala @@ -0,0 +1,81 @@ +package code.api.v6_0_0 + +import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.util.ApiRole +import code.entitlement.Entitlement +import org.json4s.native.Serialization.write + +/** + * `POST /obp/v6.0.0/management/dynamic-resource-docs/validate` must reject an unsupported + * `programming_lang` the same way `POST .../dynamic-resource-docs` (create) does, rather than + * reporting `valid = true` for a language create would actually 400 on -- see + * Http4s600.validateDynamicResourceDoc's own doc comment: CompiledObjects falls through to the + * Scala compile path for any programming_lang value it doesn't recognise as Java, so a body that + * happens to be valid Scala would otherwise "validate" successfully under a bogus/misspelled + * language. + */ +class ValidateDynamicResourceDocTest extends V600ServerSetup { + + private def validateRequest = (v6_0_0_Request / "management" / "dynamic-resource-docs" / "validate").POST <@ (user1) + + private def docWith(programmingLang: String) = + SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + programmingLang = programmingLang + ) + + feature("Validate Dynamic Resource Doc rejects an unsupported programming_lang") { + scenario("An unsupported programming_lang is rejected, not silently validated as Scala") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val resp = makePostRequest(validateRequest, write(docWith("Python"))) + + Then("the request is rejected with 400, not a 200 valid=true/false body") + resp.code should equal(400) + resp.body.toString should include("OBP-40049") + } + + scenario("programming_lang \"Scala\" is accepted (baseline, unaffected by the language check)") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val resp = makePostRequest(validateRequest, write(docWith("Scala"))) + + Then("the request reaches the compile step and responds 200") + resp.code should equal(200) + (resp.body \ "valid").values should equal(true) + } + + scenario("programming_lang \"Java\" is accepted (baseline, unaffected by the language check)") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val javaBody = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class ValidateEndpointJavaProbe implements Supplier> { + | private Object apply(Object[] args) { + | Map response = new LinkedHashMap<>(); + | response.put("greeting", "hello"); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + val doc = docWith("Java").copy(methodBody = java.net.URLEncoder.encode(javaBody, "UTF-8")) + val resp = makePostRequest(validateRequest, write(doc)) + + Then("the request reaches the compile step and responds 200") + resp.code should equal(200) + (resp.body \ "valid").values should equal(true) + } + } +}