From e686781f871dd98261f0cbec1e1d83562b2184d3 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 25 Aug 2026 14:13:19 +0200 Subject: [PATCH 01/59] fix: a sandbox IBAN identifies one account, so no two may share one The shipped fixtures could not be imported into a fresh database at all: POST /obp/v2.1.0/sandbox/data-import answered 400 "Cannot import the sandbox data" on both example_import.json files. Seven IBANs in the first and one in the second were each attached to two accounts at two different banks, and createAccounts rejects a duplicate IBAN. The rejection is right. An IBAN is globally unique by ISO 13616 -- the bank identifier is encoded inside the string, so two banks cannot hold the same one -- and OBP depends on that rather than merely assuming it. LocalMappedConnector's getBankAccountByRoutingLegacy refuses outright when a routing address matches more than one account ("Routing MUST be unique"), and that is the lookup a payment destination resolves through: BulkPaymentHandler and three v7.0.0 transaction paths all call it with bankId = None. Admitting a duplicate would not produce a working account, it would produce one that any global-routing payment then fails on, far from the import that caused it. So the data was wrong, not the check. The strings were not IBANs in the first place: 27 characters where Bosnia's format is 20, and mod-97 values of 36, 52, 50, 65, 79, 57, 45 and 0 where a valid IBAN gives 1. All sixteen are regenerated here as structurally valid, globally unique numbers, with a distinct bank code per bank -- which is the actual mechanism behind that uniqueness rather than a suffix bolted on to make the strings differ. The suites that normally cover sandbox import run against a clone of an existing database, where the accounts already exist and the duplicate branch is never the one that fires; only a genuinely fresh database reaches it. Hence a new test for the direction that had none: two accounts at different banks sharing an IBAN must be rejected, and the same two import cleanly once their IBANs differ. --- .../2016-04-28/example_import.json | 4 +- .../sandbox/example_data/example_import.json | 28 +++++------ .../api/v2_1_0/SandboxDataLoadingTest.scala | 47 +++++++++++++++++++ 3 files changed, 63 insertions(+), 16 deletions(-) diff --git a/obp-api/src/main/scala/code/api/sandbox/example_data/2016-04-28/example_import.json b/obp-api/src/main/scala/code/api/sandbox/example_data/2016-04-28/example_import.json index f6ee011d8f..96ed1ffe99 100644 --- a/obp-api/src/main/scala/code/api/sandbox/example_data/2016-04-28/example_import.json +++ b/obp-api/src/main/scala/code/api/sandbox/example_data/2016-04-28/example_import.json @@ -64,7 +64,7 @@ "currency":"GBP", "amount":"8084.32" }, - "IBAN":"BA12 1234 5123 4513 7599 6969 977", + "IBAN":"BA463990000000000001", "owners":["robert.xuk.x@example.com"], "generate_public_view":false, "generate_accountants_view":true, @@ -79,7 +79,7 @@ "currency":"GBP", "amount":"8084.32" }, - "IBAN":"BA12 1234 5123 4513 7599 6969 977", + "IBAN":"BA924990000000000001", "owners":["robert.yuk.y@example.com"], "generate_public_view":false, "generate_accountants_view":true, diff --git a/obp-api/src/main/scala/code/api/sandbox/example_data/example_import.json b/obp-api/src/main/scala/code/api/sandbox/example_data/example_import.json index 0126ddd054..85c2f9dcd1 100644 --- a/obp-api/src/main/scala/code/api/sandbox/example_data/example_import.json +++ b/obp-api/src/main/scala/code/api/sandbox/example_data/example_import.json @@ -64,7 +64,7 @@ "currency":"GBP", "amount":"6599.63" }, - "IBAN":"BA12 1234 5123 4518 4490 1189 877", + "IBAN":"BA511990000000000001", "owners":["Susan.X.0.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -79,7 +79,7 @@ "currency":"GBP", "amount":"6379.63" }, - "IBAN":"BA12 1234 5123 4511 8754 4625 177", + "IBAN":"BA241990000000000002", "owners":["Robert.X.0.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -94,7 +94,7 @@ "currency":"GBP", "amount":"7588.25" }, - "IBAN":"BA12 1234 5123 4510 4337 1399 677", + "IBAN":"BA941990000000000003", "owners":["Anil.X.0.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -109,7 +109,7 @@ "currency":"GBP", "amount":"6662.05" }, - "IBAN":"BA12 1234 5123 4514 4440 2184 977", + "IBAN":"BA671990000000000004", "owners":["Robert.X.0.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -124,7 +124,7 @@ "currency":"GBP", "amount":"3748.57" }, - "IBAN":"BA12 1234 5123 4518 9534 3427 277", + "IBAN":"BA401990000000000005", "owners":["Ellie.X.0.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -139,7 +139,7 @@ "currency":"GBP", "amount":"15860.50" }, - "IBAN":"BA12 1234 5123 4512 1957 2301 577", + "IBAN":"BA131990000000000006", "owners":["Anil.X.0.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -154,7 +154,7 @@ "currency":"GBP", "amount":"7724.41" }, - "IBAN":"BA12 1234 5123 4512 6914 8586 977", + "IBAN":"BA831990000000000007", "owners":["Anil.X.0.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -169,7 +169,7 @@ "currency":"GBP", "amount":"6599.63" }, - "IBAN":"BA12 1234 5123 4518 4490 1189 877", + "IBAN":"BA972990000000000001", "owners":["Susan.Y.9.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -184,7 +184,7 @@ "currency":"GBP", "amount":"6379.63" }, - "IBAN":"BA12 1234 5123 4511 8754 4625 177", + "IBAN":"BA702990000000000002", "owners":["Robert.Y.9.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -199,7 +199,7 @@ "currency":"GBP", "amount":"7588.25" }, - "IBAN":"BA12 1234 5123 4510 4337 1399 677", + "IBAN":"BA432990000000000003", "owners":["Anil.Y.9.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -214,7 +214,7 @@ "currency":"GBP", "amount":"6662.05" }, - "IBAN":"BA12 1234 5123 4514 4440 2184 977", + "IBAN":"BA162990000000000004", "owners":["Robert.Y.9.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -229,7 +229,7 @@ "currency":"GBP", "amount":"3748.57" }, - "IBAN":"BA12 1234 5123 4518 9534 3427 277", + "IBAN":"BA862990000000000005", "owners":["Ellie.Y.9.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -244,7 +244,7 @@ "currency":"GBP", "amount":"15860.50" }, - "IBAN":"BA12 1234 5123 4512 1957 2301 577", + "IBAN":"BA592990000000000006", "owners":["Anil.Y.9.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -259,7 +259,7 @@ "currency":"GBP", "amount":"7724.41" }, - "IBAN":"BA12 1234 5123 4512 6914 8586 977", + "IBAN":"BA322990000000000007", "owners":["Anil.Y.9.GH"], "generate_public_view":true, "generate_accountants_view":true, diff --git a/obp-api/src/test/scala/code/api/v2_1_0/SandboxDataLoadingTest.scala b/obp-api/src/test/scala/code/api/v2_1_0/SandboxDataLoadingTest.scala index 646fdb18de..ae78699734 100644 --- a/obp-api/src/test/scala/code/api/v2_1_0/SandboxDataLoadingTest.scala +++ b/obp-api/src/test/scala/code/api/v2_1_0/SandboxDataLoadingTest.scala @@ -1082,6 +1082,53 @@ class SandboxDataLoadingTest extends FlatSpec with SendServerRequests with Match Connector.connector.vend.getBankAccountLegacy(BankId(acc1.bank), AccountId(acc2.id), None).isDefined should equal(true) } + it should "not allow two accounts at DIFFERENT banks to share an IBAN either" in { + // The global-uniqueness half of the rule, which had no test at all. + // + // An IBAN is globally unique by ISO 13616 -- the bank identifier is encoded INSIDE the + // string, so two banks cannot legitimately hold the same one. OBP does not merely assume + // that; it depends on it. LocalMappedConnector.getBankAccountByRoutingLegacy, called with + // no bankId, refuses outright when a routing address matches more than one account: + // + // if (routing.size > 1) { // Routing MUST be unique + // Failure(s"$AccountRoutingNotUnique (scheme: $scheme, address: $address)") + // + // and that is the lookup PAYMENT DESTINATIONS resolve through -- BulkPaymentHandler and + // three v7.0.0 transaction paths all call it with bankId = None. So letting a duplicate in + // at import time does not create a working account: it creates one that any global-routing + // payment then fails on, far from the import that caused it. + // + // Rejecting at import is therefore the correct behaviour, and this test exists so nobody + // "fixes" the duplicate check by scoping it per bank to make a broken fixture load. + val users = standardUsers + val banks = standardBanks + + def getResponse(accountJsons : List[JValue]) = { + BankAccountRouting.bulkDelete_!!() + val json = createImportJson(banks.map(Extraction.decompose), users.map(Extraction.decompose), accountJsons, Nil, Nil, Nil, Nil, Nil) + postImportJson(json) + } + + val accAtBank1 = account1AtBank1 + val accAtBank2 = account1AtBank2 + + val bank1Json = Extraction.decompose(accAtBank1) + // Same IBAN, different bank. Nothing else changed. + val bank2SameIbanJson = replaceField(Extraction.decompose(accAtBank2), "IBAN", accAtBank1.IBAN) + + getResponse(List(bank1Json, bank2SameIbanJson)).code should equal(FAILED) + + // And nothing partially imported. + Connector.connector.vend.getBankAccountLegacy(BankId(accAtBank1.bank), AccountId(accAtBank1.id), None).isDefined should equal(false) + Connector.connector.vend.getBankAccountLegacy(BankId(accAtBank2.bank), AccountId(accAtBank2.id), None).isDefined should equal(false) + + // The same two accounts import fine once their IBANs differ -- proving the rejection above + // was about the IBAN collision and not about anything else in the payload. + getResponse(List(bank1Json, Extraction.decompose(accAtBank2))).code should equal(SUCCESS) + Connector.connector.vend.getBankAccountLegacy(BankId(accAtBank1.bank), AccountId(accAtBank1.id), None).isDefined should equal(true) + Connector.connector.vend.getBankAccountLegacy(BankId(accAtBank2.bank), AccountId(accAtBank2.id), None).isDefined should equal(true) + } + it should "not allow an account to be created with an existing IBAN" in { val banks = standardBanks.map(Extraction.decompose) val users = standardUsers.map(Extraction.decompose) From 4a5e3206e67f009cf4d1c1285e1c255dc0fa40a4 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 25 Aug 2026 14:13:58 +0200 Subject: [PATCH 02/59] test: sweep every endpoint for the auth it declares, and for crashes Of the 850 endpoints a caller can reach, 384 are referenced by no test at all, and of those that are tested only about a third carry an anonymous-access scenario. Writing the rest by hand is several hundred near-identical files that then rot one endpoint at a time; driving them off the ResourceDoc registry means an endpoint added tomorrow is swept the day it is registered. Three sweeps, each asserting something the others cannot see: AuthSweepTest anonymous call is refused where the doc says it must be, and is NOT refused where the doc says it is public; a role-gated endpoint refuses a user holding no entitlements FailureSweepTest a fully-entitled caller asking for something that does not exist gets 4xx, never 5xx SuccessSweepTest the endpoints that need no setup at all actually answer EndpointCatalog is the single place that answers "what exists and what does each one claim", so that SweepCoverageTest can check an exact identity: swept plus skipped equals the catalog, with every skip carrying one of three enumerated reasons. There is no fourth bucket for an endpoint to fall into quietly. Three things about the source data are easy to get wrong and all three are load-bearing, so they are written down in EndpointCatalog rather than discovered again later. The docs live on the Http4s objects; every APIMethods*.scala is now a stub whose Lift registrations are commented out, so reading those files for a catalog finds nothing. "Needs authentication" is derived, not declared, and the predicate has to be evaluated on the constructed ResourceDoc because the constructor rewrites errorResponseBodies and several docs compute theirs from a prop. And not every ALL_CAPS URL segment is a placeholder -- SANDBOX_TAN and EMAIL are real literals, so substitution is driven by what the name says it is rather than by a copy of the framework's private literal list. Requests run in-process against Http4sApp.httpApp: no TCP, no server startup, single-digit milliseconds each. Scenarios are one per version rather than one per endpoint because ServerSetupWithTestData rebuilds its fixtures for every scenario and at ~1600 assertions that cost, not the assertions, would dominate. Each scenario collects every mismatch and fails once with the whole list. --- .../scala/code/api/sweep/AuthSweepTest.scala | 191 +++++++++++++++++ .../code/api/sweep/EndpointCatalog.scala | 137 +++++++++++++ .../code/api/sweep/FailureSweepTest.scala | 194 ++++++++++++++++++ .../code/api/sweep/SuccessSweepTest.scala | 187 +++++++++++++++++ .../code/api/sweep/SweepCoverageTest.scala | 149 ++++++++++++++ 5 files changed, 858 insertions(+) create mode 100644 obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala create mode 100644 obp-api/src/test/scala/code/api/sweep/EndpointCatalog.scala create mode 100644 obp-api/src/test/scala/code/api/sweep/FailureSweepTest.scala create mode 100644 obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala create mode 100644 obp-api/src/test/scala/code/api/sweep/SweepCoverageTest.scala diff --git a/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala b/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala new file mode 100644 index 0000000000..285cac71a9 --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala @@ -0,0 +1,191 @@ +package code.api.sweep + +import cats.effect.IO +import cats.effect.unsafe.IORuntime +import code.api.util.APIUtil.ResourceDoc +import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, UserHasMissingRoles} +import code.api.util.http4s.Http4sApp +import code.setup.{DefaultUsers, ServerSetupWithTestData} +import fs2.Stream +import org.http4s.{Header, Headers, Method, Request, Uri} +import org.json4s.JValue +import org.json4s.JsonAST.JObject +import org.scalatest.Tag +import org.typelevel.ci.CIString +import com.openbankproject.commons.util.JsonAliases.parse + +/** + * Every endpoint answers an unauthenticated call the way its own ResourceDoc says it will. + * + * Why a sweep instead of more hand-written suites: of the 870 endpoints a caller can reach, + * 384 are referenced by no test at all, and of the ones that ARE tested only about a third + * carry an anonymous-access scenario. Writing those by hand is several hundred near-identical + * files that then rot one endpoint at a time; driving them off the registry means an endpoint + * added tomorrow is swept the day it is registered, and SweepCoverageTest fails if it is not. + * + * Three assertions, chosen by what the doc declares: + * + * auth required, no roles anonymous -> 401 with exactly AuthenticatedUserIsRequired + * auth required, roles anonymous -> 401; authenticated-without-the-role -> 403 + * public anonymous -> anything EXCEPT 401 + * + * The public case is deliberately weak. A public endpoint may well answer 400 or 404 to a call + * with no body and a nonexistent id — that is not an authentication defect, and asserting 200 + * would make this sweep a fixture problem instead of an auth check. What must never happen is a + * public endpoint demanding credentials, and that is what is asserted. + * + * The 403 assertion uses startWith, not equal: at runtime the message carries the missing roles + * joined with " or ", and ApiRole.requiresBankId appends " for BankId(...)". An equality + * assertion here passes locally and fails the moment an endpoint gains a second role. + * + * ── Why one scenario per version rather than per endpoint ── + * ServerSetupWithTestData.beforeEach wipes and rebuilds banks, accounts and views for EVERY + * scenario. At ~1600 assertions that fixture cost, not the assertions, would dominate: the + * requests themselves run in-process against Http4sApp.httpApp with no TCP and no server, and + * cost single-digit milliseconds each. So each scenario sweeps one version, collects every + * mismatch, and fails once with the whole list. The per-endpoint detail that a + * scenario-per-endpoint layout would have given is preserved in that list — each line names the + * operationId, the verb, the URL, the expectation and what actually came back. + */ +class AuthSweepTest extends ServerSetupWithTestData with DefaultUsers { + + object AuthSweep extends Tag("AuthSweep") + + implicit val runtime: IORuntime = IORuntime.global + private lazy val app = Http4sApp.httpApp + + /** One in-process request. No TCP, no server startup. */ + private def call(verb: String, path: String, headers: Map[String, String]): (Int, JValue) = { + val method = Method.fromString(verb.toUpperCase).getOrElse(Method.GET) + val req = Request[IO]( + method = method, + uri = Uri.unsafeFromString(path), + headers = Headers(headers.map { case (k, v) => Header.Raw(CIString(k), v) }.toList), + body = Stream.empty + ) + val resp = app.run(req).unsafeRunSync() + val bodyStr = resp.bodyText.compile.string.unsafeRunSync() + val json = try { if (bodyStr.trim.isEmpty) JObject(Nil) else parse(bodyStr) } + catch { case _: Exception => JObject(Nil) } + (resp.status.code, json) + } + + private def messageOf(json: JValue): String = { + implicit val formats = code.api.util.CustomJsonFormats.formats + (json \ "message").extractOpt[String].getOrElse("") + } + + /** A token for a user holding no entitlements at all — the natural 403 probe. */ + private def noRoleHeaders: Map[String, String] = Map("DirectLogin" -> s"token=${token1.value}") + + /** + * Real identifiers from the fixtures, for the role assertion only. + * + * An endpoint that declares BankNotFound and carries BANK_ID validates the bank before it + * checks roles, so a nonexistent bank answers 404 and the role gate never runs. These come + * from the fixture banks/accounts ServerSetupWithTestData creates, read directly rather than + * over HTTP — the sweep is in-process and a round trip per lookup would be the only slow part + * of it. + */ + private lazy val realEntities: Map[String, String] = { + val bank = code.bankconnectors.LocalMappedConnector.getBanksLegacy(None).map(_._1).getOrElse(Nil).headOption + bank match { + case Some(b) => + val accountId = code.model.dataAccess.MappedBankAccount + .find(net.liftweb.mapper.By(code.model.dataAccess.MappedBankAccount.bank, b.bankId.value)) + .map(_.accountId.value) + Map("BANK_ID" -> b.bankId.value) ++ accountId.map("ACCOUNT_ID" -> _).toList.toMap + case None => Map.empty + } + } + + private def describe(doc: ResourceDoc): String = + s"${doc.operationId} ${doc.requestVerb} ${EndpointCatalog.concretePath(doc)}" + + // ── the three checks, each returning a failure line or None ────────────────── + + private def checkAnonymousIs401(doc: ResourceDoc): Option[String] = { + val (code, json) = call(doc.requestVerb, EndpointCatalog.concretePath(doc), Map.empty) + if (code != 401) + Some(s"${describe(doc)} -- expected 401 for an anonymous call, got $code") + else if (messageOf(json) != AuthenticatedUserIsRequired) + Some(s"${describe(doc)} -- 401 but message was '${messageOf(json)}', expected '$AuthenticatedUserIsRequired'") + else None + } + + private def checkPublicIsNot401(doc: ResourceDoc): Option[String] = { + val (code, _) = call(doc.requestVerb, EndpointCatalog.concretePath(doc), Map.empty) + if (code == 401) + Some(s"${describe(doc)} -- declares no authentication requirement yet answered 401 anonymously") + else None + } + + private def checkNoRoleIs403(doc: ResourceDoc): Option[String] = { + val path = EndpointCatalog.concretePath(doc, realEntities) + val (code, json) = call(doc.requestVerb, path, noRoleHeaders) + val roles = doc.roles.getOrElse(Nil).map(_.toString).mkString(",") + if (code != 403) + Some(s"${doc.operationId} ${doc.requestVerb} $path -- roles $roles: " + + s"expected 403 for a user holding no entitlements, got $code") + else if (!messageOf(json).startsWith(UserHasMissingRoles)) + Some(s"${doc.operationId} ${doc.requestVerb} $path -- 403 but message was " + + s"'${messageOf(json)}', expected it to start with '$UserHasMissingRoles'") + else None + } + + // ── the sweep, one scenario per version ───────────────────────────────────── + + private lazy val byVersion: Map[String, List[ResourceDoc]] = + EndpointCatalog.all + .filter(EndpointCatalog.skipReason(_).isEmpty) + .groupBy(_.implementedInApiVersion.toString) + + feature("Every reachable endpoint enforces the authentication its ResourceDoc declares") { + + byVersion.keys.toList.sorted.foreach { version => + scenario(s"$version -- anonymous calls are refused, public ones are not", AuthSweep) { + // Endpoint-level enable/disable props are read per request by ResourceDocMiddleware, + // and a disabled endpoint falls through to 404 rather than 401 -- which would read as a + // sweep failure. Cleared the way SwaggerDocsTest does; PropsReset restores afterwards. + // Written out in each scenario rather than shared in a helper because + // check_test_isolation.py scans statically: any setPropsValues outside a scenario body + // reads to it as a class-body push, `def` or not. + setPropsValues("api_disabled_endpoints" -> "[]", "api_enabled_endpoints" -> "[]") + val docs = byVersion(version) + + When(s"every one of the ${docs.size} $version endpoints is called with no credentials") + val failures = docs.flatMap { doc => + if (EndpointCatalog.needsAuthentication(doc)) checkAnonymousIs401(doc) + else checkPublicIsNot401(doc) + } + + Then("each one answers as its own ResourceDoc declares") + withClue(s"${failures.size} of ${docs.size} $version endpoints disagreed with their own " + + s"ResourceDoc:\n${failures.mkString("\n")}\n") { + failures shouldBe empty + } + } + } + + byVersion.keys.toList.sorted.foreach { version => + lazy val roleGated = byVersion(version) + .filter(EndpointCatalog.isRoleGated) + .filter(EndpointCatalog.roleSkipReason(_).isEmpty) + + if (roleGated.nonEmpty) { + scenario(s"$version -- role-gated endpoints refuse a user holding no entitlements", AuthSweep) { + setPropsValues("api_disabled_endpoints" -> "[]", "api_enabled_endpoints" -> "[]") + + When(s"every one of the ${roleGated.size} role-gated $version endpoints is called as a user with no entitlements") + val failures = roleGated.flatMap(checkNoRoleIs403) + + Then("each one answers 403 naming the roles it wanted") + withClue(s"${failures.size} of ${roleGated.size} role-gated $version endpoints did not " + + s"refuse an unentitled user:\n${failures.mkString("\n")}\n") { + failures shouldBe empty + } + } + } + } + } +} diff --git a/obp-api/src/test/scala/code/api/sweep/EndpointCatalog.scala b/obp-api/src/test/scala/code/api/sweep/EndpointCatalog.scala new file mode 100644 index 0000000000..9e1af85e14 --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/EndpointCatalog.scala @@ -0,0 +1,137 @@ +package code.api.sweep + +// EndpointAuthMode and its four cases are members of the APIUtil object, not of the package. +import code.api.util.APIUtil.{ResourceDoc, UserOnly} +import code.api.util.ErrorMessages.$AuthenticatedUserIsRequired +import code.api.util.ApiTag + +/** + * The one place that answers "what endpoints exist, and what does each one claim about auth". + * + * Every sweep in this package reads the catalog from here rather than assembling its own, so + * that the coverage identity in SweepCoverageTest is checkable: asserted + skipped == catalog, + * with no third bucket anyone can quietly slip an endpoint into. + * + * Three things about the source data are easy to get wrong, and all three are load-bearing: + * + * 1. The docs live on the Http4s objects, NOT on APIMethods*. Every APIMethods{121..600}.scala + * is now a stub whose entire body is `val ImplementationsX = Http4sX.ImplementationsX` — the + * Lift registrations below it are commented out. Reading those files for a catalog finds + * nothing. Http4s700.allResourceDocs is the aggregate: every version's docs, deduplicated by + * (requestUrl, requestVerb) keeping the newest, which is exactly the set a caller can reach. + * + * 2. "Needs authentication" is derived, not declared. There is no flag on ResourceDoc; the + * middleware's own predicate is + * errorResponseBodies.contains($AuthenticatedUserIsRequired) || roles.exists(_.nonEmpty) + * (ResourceDocMiddleware.needsAuthentication). We reproduce it here rather than approximate it. + * + * 3. That predicate has to be evaluated on the CONSTRUCTED ResourceDoc, never on the source text. + * The constructor rewrites errorResponseBodies: it appends AuthenticatedUserIsRequired and + * UserHasMissingRoles when roles are present, and adds/removes AuthenticatedUserIsRequired + * based on the description. Several docs also compute their error list from a prop — e.g. + * getApiProduct branches on getApiProductsIsPublic — so the answer depends on the props in + * force at the moment the sweep runs, which is another reason to ask the object and not a grep. + */ +object EndpointCatalog { + + /** Every endpoint a caller can reach, newest version of each (url, verb). */ + def all: List[ResourceDoc] = code.api.v7_0_0.Http4s700.allResourceDocs.toList + + /** The middleware's own rule, reproduced. See ResourceDocMiddleware.needsAuthentication. */ + def needsAuthentication(doc: ResourceDoc): Boolean = + doc.errorResponseBodies.contains($AuthenticatedUserIsRequired) || doc.roles.exists(_.nonEmpty) + + def isRoleGated(doc: ResourceDoc): Boolean = doc.roles.exists(_.nonEmpty) + + /** + * Why an endpoint is not swept. Every exclusion is one of these — a sweep may not invent a + * reason inline, because SweepCoverageTest counts these and nothing else. + */ + sealed abstract class SkipReason(val why: String) + case object DynamicDoc extends SkipReason( + "tagged apiTagDynamic: created per-database, so its presence is machine state, not contract") + case object NonUserAuthMode extends SkipReason( + "authMode is not UserOnly: an anonymous call may legitimately not be 401 " + + "(ApplicationOnly drops AuthenticatedUserIsRequired entirely). EndpointAuthModeTest covers these.") + case object AutoValidateRolesOff extends SkipReason( + "disableAutoValidateRoles(): roles stay in the doc for the catalog but the framework does " + + "not enforce them, so asserting 403 would assert something no code promises") + + /** Skip reason, or None when the endpoint is in scope for the auth sweep. */ + def skipReason(doc: ResourceDoc): Option[SkipReason] = + if (doc.tags.contains(ApiTag.apiTagDynamic)) Some(DynamicDoc) + else if (doc.authMode != UserOnly) Some(NonUserAuthMode) + else None + + /** Skip reason for the role dimension specifically — a superset of skipReason. */ + def roleSkipReason(doc: ResourceDoc): Option[SkipReason] = + skipReason(doc).orElse( + if (!doc.isAutoValidateRoles) Some(AutoValidateRolesOff) else None) + + /** + * Not every ALL_CAPS segment in a requestUrl is a placeholder. OBP serves real literals in + * that shape — `/transaction-request-types/SANDBOX_TAN/`, `/my/consents/EMAIL` — and + * substituting those produces a URL that routes nowhere, which reads as a 404/400 "auth + * failure" that is entirely the sweep's own doing. The first run of AuthSweepTest hit exactly + * that on nine endpoints across v2.1.0 and v3.1.0. + * + * The production rule lives in ResourceDocMatcher.isTemplateVariable, which consults a private + * `literalAllCapsSegments` set. Copying that set here would give us a second copy to keep in + * sync, and a stale copy fails in the direction that is hardest to notice — a literal newly + * added there would be substituted here and the sweep would quietly stop covering that path. + * + * So this asks the opposite question: rather than "is it a literal", "do I have a value for + * it". A segment is substituted only when its NAME says it is an id or a code. That happens to + * separate the two sets exactly, including the pairs that differ by suffix alone — CARD and + * ACCOUNT are literals, CARD_ID and ACCOUNT_ID are placeholders — and it needs no maintenance + * when a new literal appears, because an unrecognised segment is left alone by default. + */ + private def isPlaceholder(seg: String): Boolean = + seg.nonEmpty && + seg == seg.toUpperCase && + seg.forall(c => c.isLetter || c == '_' || c.isDigit) && + // `ID` rather than `_ID`: the UK Open Banking paths spell them without the separator + // (CONSENTID, ACCOUNTID, BASKETID, DOMESTICPAYMENTID …), and leaving those literal sent + // the string "CONSENTID" to the server as though it were an id. + (seg.endsWith("ID") || seg.endsWith("_CODE") || seg.endsWith("_NAME") || + seg == "PROVIDER" || seg == "USERNAME" || seg == "USER_EMAIL") + + // Checked against Http4sSupport's literalAllCapsSegments: not one of the sixteen ends in ID, + // _CODE or _NAME, so the rule above separates the two sets cleanly. + // + // Known limitation, stated rather than papered over: a handful of ALL_CAPS segments are + // neither — PAYMENT_SERVICE and PAYMENT_PRODUCT take enumerated Berlin Group values, + // SCA_METHOD likewise, and CARDANO / MOBILE_WALLET / ETH_SEND_TRANSACTION are literals this + // list does not carry. Those stay verbatim, which routes them to the endpoint that expects + // that literal (correct for the literals) or produces a 404 (visible, and safe: a sweep + // reporting a 404 it caused is a nuisance, one silently substituting over a real literal is + // a coverage hole). + + /** + * The concrete path to call. + * + * `entities` supplies values for the identifiers the caller wants resolvable. Anything not + * named there gets a well-formed value that does not exist. + * + * The distinction matters for the 403 assertion. A ResourceDoc that declares BankNotFound and + * carries BANK_ID is validated for bank existence BEFORE its roles are checked + * (APIUtil.ResourceDoc's isNeedCheckBank), so a nonexistent bank answers 404 and the role gate + * is never reached. The sweep's first run read those 404s as missing 403s across some thirty + * endpoints; they were the entity check doing its job. Passing a real bank id is what makes + * the assertion actually about roles. + */ + def concretePath(doc: ResourceDoc, entities: Map[String, String] = Map.empty): String = { + val segments = doc.requestUrl.split("/").map { seg => + if (isPlaceholder(seg)) entities.getOrElse(seg, defaultValue(seg)) else seg + } + "/obp/" + doc.implementedInApiVersion.apiShortVersion + segments.mkString("/") + } + + /** Well-formed, and deliberately not present. */ + private def defaultValue(seg: String): String = seg match { + case "ACCOUNT_ID" | "USER_ID" => "00000000-0000-0000-0000-000000000000" + case "VIEW_ID" => "owner" + case "USER_EMAIL" => "sweep-no-such-user@example.com" + case _ => "sweep-no-such-" + seg.toLowerCase.replace('_', '-') + } +} diff --git a/obp-api/src/test/scala/code/api/sweep/FailureSweepTest.scala b/obp-api/src/test/scala/code/api/sweep/FailureSweepTest.scala new file mode 100644 index 0000000000..38c15abd75 --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/FailureSweepTest.scala @@ -0,0 +1,194 @@ +package code.api.sweep + +import cats.effect.IO +import cats.effect.unsafe.IORuntime +import code.api.util.APIUtil.ResourceDoc +import code.api.util.{ApiRole, CustomJsonFormats} +import code.api.util.http4s.Http4sApp +import code.entitlement.Entitlement +import code.setup.{DefaultUsers, ServerSetupWithTestData} +import fs2.Stream +import org.http4s.{Header, Headers, Method, Request, Uri} +import org.json4s.Extraction +import org.json4s.JValue +import org.json4s.JsonAST.JObject +import org.json4s.native.JsonMethods.{compact, render} +import org.scalatest.Tag +import org.typelevel.ci.CIString +import com.openbankproject.commons.util.JsonAliases.parse + +/** + * No endpoint answers a well-formed request with a 500. + * + * The auth sweep proves an endpoint refuses the wrong caller. This one proves it survives the + * right caller asking for something that is not there — which is the far more common shape of a + * production incident, and the one a migration is most likely to introduce. A connector that + * returns a slightly different empty value, a JSON codec that stops handling a null, a column + * read that no longer tolerates NULL: none of those show up as a wrong answer, they show up as + * a 500 on a request that used to work. + * + * That class of defect has a track record on this codebase. The first run of AuthSweepTest found + * `createTransactionRequestFreeForm` answering 500 to a nonexistent view id where it should have + * answered 403 or 404, and it found it by accident — the sweep was looking for something else. + * This suite looks for it on purpose, across every endpoint. + * + * ── What is sent ── + * + * A caller holding EVERY role, so nothing stops at the authorisation gate; path identifiers that + * are well-formed and do not exist; and, for verbs that take one, the endpoint's own + * `exampleRequestBody` serialised through the same json4s path the server uses to publish it. + * + * The example body is the right instrument here for the reason it is the wrong one for a success + * test: it is structurally valid and referentially meaningless. Its bank ids, currencies and + * entity references are illustrative, so an endpoint accepting it will almost always answer 400 + * or 404 — which is exactly the input that exercises the error paths, and exactly where an + * unhandled null or an over-narrow match arm turns into a 500. + * + * ── What is asserted ── + * + * 4xx fine, whatever the code. "Not found", "bad request", "not allowed" are all correct + * answers to a request for something that does not exist. + * 2xx fine. Some endpoints legitimately succeed with no arguments, or answer an empty list. + * 5xx a finding, always. + * + * Nothing here asserts WHICH 4xx. That would be a contract test, and the contract suite already + * owns it; asserting it twice, from a place with no baseline to compare against, would produce + * failures every time a message was reworded. + */ +class FailureSweepTest extends ServerSetupWithTestData with DefaultUsers { + + object FailureSweep extends Tag("FailureSweep") + + implicit val runtime: IORuntime = IORuntime.global + private lazy val app = Http4sApp.httpApp + + /** + * A caller holding every role in the system. + * + * Granted directly through the Entitlement provider rather than over the API — the same thing + * 161 existing test files do — because the goal is to get PAST authorisation, not to test it. + * Without this the sweep would stop at 403 on the ~478 role-gated endpoints and never reach + * the code that might crash. + */ + private def omniscientUser: Map[String, String] = { + ApiRole.availableRoles.foreach { role => + // Bank-scoped roles need a bank; system-wide ones must be granted with an empty bankId. + // valueOf throws on a name it does not recognise, and availableRoles includes dynamic + // roles whose backing entity may not exist in this database -- a grant that cannot be + // made is not a reason to abandon the other several hundred. + try { + val bankId = if (ApiRole.valueOf(role).requiresBankId) realBankId.getOrElse("") else "" + Entitlement.entitlement.vend.addEntitlement(bankId, resourceUser1.userId, role) + } catch { case _: Exception => () } + } + Map("DirectLogin" -> s"token=${token1.value}") + } + + private def realBankId: Option[String] = + code.bankconnectors.LocalMappedConnector.getBanksLegacy(None) + .map(_._1).getOrElse(Nil).headOption.map(_.bankId.value) + + private def entities: Map[String, String] = + realBankId.map("BANK_ID" -> _).toList.toMap + + private def call(verb: String, path: String, headers: Map[String, String], body: String) + : (Int, JValue) = { + val method = Method.fromString(verb.toUpperCase).getOrElse(Method.GET) + val hdrs = if (body.nonEmpty) headers + ("Content-Type" -> "application/json") else headers + val req = Request[IO]( + method = method, + uri = Uri.unsafeFromString(path), + headers = Headers(hdrs.map { case (k, v) => Header.Raw(CIString(k), v) }.toList), + body = if (body.nonEmpty) Stream.emits(body.getBytes("UTF-8")).covary[IO] else Stream.empty + ) + val resp = app.run(req).unsafeRunSync() + val bodyStr = resp.bodyText.compile.string.unsafeRunSync() + val json = try { if (bodyStr.trim.isEmpty) JObject(Nil) else parse(bodyStr) } + catch { case _: Exception => JObject(Nil) } + (resp.status.code, json) + } + + /** + * The doc's own example body as JSON, or "" when it has none. + * + * Extraction.decompose under CustomJsonFormats is the same route Http4s uses to publish these + * objects, so what is sent is what the documentation shows a caller to send. + */ + private def exampleBody(doc: ResourceDoc): String = { + implicit val formats = CustomJsonFormats.formats + doc.exampleRequestBody match { + case null => "" + case body => + try compact(render(Extraction.decompose(body))) catch { case _: Exception => "" } + } + } + + /** + * Endpoints whose 5xx is the answer they are built to give. + * + * Exactly one so far, and it is not a real endpoint: Http4s700 registers + * `POST /obp/v7.0.0/test/rollback-check` inside `if (Props.testMode)` specifically to abort a + * transaction and prove the rollback happened, so a 500 is its pass condition. It exists only + * under `run.mode=test`, which is to say only where this sweep runs. + * + * Kept as a named map rather than removed from the catalog: SweepCoverageTest counts what the + * sweeps cover, and an endpoint quietly dropped from a list is the failure mode that whole + * test exists to prevent. Anything added here needs the same kind of reason. + */ + private val expected5xx: Map[String, String] = Map( + "OBPv7.0.0-testRollbackEndpoint" -> ("test-mode-only endpoint that deliberately throws to " + + "verify transaction rollback; its 500 IS the assertion (Http4s700, Props.testMode)"), + "OBPv7.0.0-createTestEmail" -> ("500 OBP-10056: refuses to send because portal_external_url " + + "is unset, which is true of any test rig. Environmental -- but a missing configuration is " + + "a 503, not a 500, so the status itself is logged in REGRESSION-GAPS rather than accepted " + + "as correct") + ) + + private lazy val inScope: List[ResourceDoc] = + EndpointCatalog.all.filter(EndpointCatalog.skipReason(_).isEmpty) + + private def check(doc: ResourceDoc, headers: Map[String, String], + ents: Map[String, String]): Option[String] = { + val path = EndpointCatalog.concretePath(doc, ents) + val body = if (doc.requestVerb.toUpperCase == "GET" || doc.requestVerb.toUpperCase == "DELETE") + "" else exampleBody(doc) + val (status, json) = call(doc.requestVerb, path, headers, body) + if (status >= 500 && !expected5xx.contains(doc.operationId)) { + implicit val formats = CustomJsonFormats.formats + val msg = (json \ "message").extractOpt[String].getOrElse("") + Some(s"${doc.operationId} ${doc.requestVerb} $path -> HTTP $status: $msg") + } else None + } + + private lazy val byVersion: Map[String, List[ResourceDoc]] = + inScope.groupBy(_.implementedInApiVersion.toString) + + feature("No endpoint answers a well-formed request with a server error") { + + byVersion.keys.toList.sorted.foreach { version => + scenario(s"$version -- a fully-entitled caller asking for something absent gets 4xx, never 5xx", + FailureSweep) { + setPropsValues("api_disabled_endpoints" -> "[]", "api_enabled_endpoints" -> "[]") + // Grant once per scenario, not once per class: beforeEach wipes the entitlement + // table, so a lazy val granted during the first scenario leaves every later one + // calling as an unentitled user -- which stops at 403 and never reaches the code + // that might crash. That is how the first run reported only two 5xx. + val headers = omniscientUser + val ents = entities + val docs = byVersion(version) + + When(s"each of the ${docs.size} $version endpoints is called with valid credentials, " + + s"every role, a nonexistent id and its own example body") + val failures = docs.flatMap(check(_, headers, ents)) + + Then("none of them crashes") + withClue(s"${failures.size} of ${docs.size} $version endpoints answered 5xx. A request " + + s"for something that does not exist is an ordinary 404; a 500 means an " + + s"unhandled path, and it is the shape a migration introduces most often:\n" + + s"${failures.mkString("\n")}\n") { + failures shouldBe empty + } + } + } + } +} diff --git a/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala b/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala new file mode 100644 index 0000000000..69fc079403 --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala @@ -0,0 +1,187 @@ +package code.api.sweep + +import cats.effect.IO +import cats.effect.unsafe.IORuntime +import code.api.util.APIUtil.ResourceDoc +import code.api.util.{ApiRole, CustomJsonFormats} +import code.api.util.http4s.Http4sApp +import code.entitlement.Entitlement +import code.setup.{DefaultUsers, ServerSetupWithTestData} +import fs2.Stream +import org.http4s.{Header, Headers, Method, Request, Uri} +import org.json4s.JValue +import org.json4s.JsonAST.JObject +import org.scalatest.Tag +import org.typelevel.ci.CIString +import com.openbankproject.commons.util.JsonAliases.parse + +/** + * The endpoints that need nothing but a caller actually answer. + * + * The other two sweeps assert what happens when something is wrong: no credentials, no role, no + * such entity. Neither would notice an endpoint that had stopped returning data altogether — a + * GET that answers 404 for every caller passes the failure sweep, which only objects to 5xx. + * This one closes that: for the endpoints where a correct answer requires no setup at all, a + * fully-entitled caller must get one. + * + * ── Why only the no-path-variable endpoints ── + * + * Of the endpoints a caller can reach, roughly a third carry no ALL_CAPS placeholder in their + * URL — /banks, /users/current, /my/accounts, /management/metrics and so on. For those, "call it + * and expect an answer" is a complete test: there is no entity to create first, so a non-answer + * is the endpoint's own fault. + * + * The remaining two thirds are deliberately out of scope here. About half of them reference an + * identifier the fixtures do supply (BANK_ID, ACCOUNT_ID, VIEW_ID …) and the other half + * reference one nothing creates — CHAT_ROOM_ID, CUSTOMER_ID, CONSENT_ID and about twenty more. + * Both need per-cluster setup to be worth asserting, and a sweep that fabricated ids for them + * would be asserting 404-handling under a name that promises success. They are the next two + * waves, not this one. + * + * ── GET only ── + * + * The 116 no-variable POSTs are excluded because a generic success POST is not a thing: the + * example bodies are illustrative rather than referentially valid (which is exactly what makes + * them good failure-path input, see FailureSweepTest), and a POST that did succeed would leave + * a row behind that the next scenario's fixture reset may or may not clear. A write-path success + * sweep needs per-endpoint bodies and per-endpoint cleanup; that is Wave 3b's problem. + * + * ── What "an answer" means ── + * + * 2xx. Not a shape, not a field — the contract suite owns field-level assertions and has a + * baseline to compare against, which this does not. Asserting shape here would duplicate that + * work from a worse position and fail every time a message was reworded. + * + * Endpoints that legitimately cannot answer 2xx on a fixture database — because they need a + * connector this build does not have, or a feature the props disable — are listed in + * `expectedNon2xx` with the reason, and asserted to STILL not be 5xx. An endpoint that stops + * answering is a finding; an endpoint that was never going to answer here is a documented skip. + */ +class SuccessSweepTest extends ServerSetupWithTestData with DefaultUsers { + + object SuccessSweep extends Tag("SuccessSweep") + + implicit val runtime: IORuntime = IORuntime.global + private lazy val app = Http4sApp.httpApp + + /** + * Endpoints that answer non-2xx on a fixture database for a stated reason. + * + * Every entry is a claim that the non-answer is environmental, not a defect. They are still + * called, and still required not to crash — the skip is only from the 2xx assertion. Keeping + * them here rather than filtering them out of the catalog means SweepCoverageTest still counts + * them, and means each exemption has to be written down next to its reason. + */ + private val expectedNon2xx: Map[String, String] = Map( + // ── needs an external service this build does not run ── + "OBPv2.0.0-elasticSearchMetrics" -> "404: needs an Elasticsearch instance; none in the test rig", + "OBPv2.0.0-elasticSearchWarehouse" -> "404: needs an Elasticsearch instance; none in the test rig", + "OBPv2.2.0-getMessageDocs" -> ("400 OBP-30211: asks which connector's message docs to " + + "return; the fixture rig runs `mapped`, which publishes none"), + "OBPv3.1.0-getObpConnectorLoopback" -> "400 OBP-10010: not implemented by the mapped connector", + "OBPv6.0.0-getMessageDocsJsonSchema" -> "same as getMessageDocs -- no connector message docs to derive a schema from", + + // ── needs a certificate the test caller does not present ── + "OBPv5.1.0-mtlsClientCertificateInfo" -> ("400 OBP-20300: reports the caller's client " + + "certificate; these requests are driven in-process with no TLS peer"), + "OBPv4.0.0-verifyRequestSignResponse" -> ("401 OBP-20311: authenticates by JWS request " + + "signature rather than by session; the sweep signs nothing"), + + // ── the URL names an entity, in a segment shaped like a literal ── + // These carry ALL_CAPS segments that EndpointCatalog deliberately leaves verbatim + // (API_COLLECTION_NAME, WEBUI_PROP_NAME, SCHEME), so the server correctly reports that no + // such entity exists. Creating one first is Wave 3c's job, not this sweep's. + "OBPv4.0.0-getMyApiCollectionByName" -> "400 OBP-30079: no ApiCollection named API_COLLECTION_NAME", + "OBPv4.0.0-getMyApiCollectionEndpoints" -> "400 OBP-30079: no ApiCollection named API_COLLECTION_NAME", + "OBPv6.0.0-getWebUiProp" -> "400 OBP-08003: no WebUi prop named WEBUI_PROP_NAME", + "OBPv7.0.0-getRoutingScheme" -> "404 OBP-30514: no routing scheme named SCHEME" + ) + + private def realBankId: Option[String] = + code.bankconnectors.LocalMappedConnector.getBanksLegacy(None) + .map(_._1).getOrElse(Nil).headOption.map(_.bankId.value) + + /** A caller holding every role — the same construction FailureSweepTest uses. */ + private def entitledCaller: Map[String, String] = { + ApiRole.availableRoles.foreach { role => + try { + val bankId = if (ApiRole.valueOf(role).requiresBankId) realBankId.getOrElse("") else "" + Entitlement.entitlement.vend.addEntitlement(bankId, resourceUser1.userId, role) + } catch { case _: Exception => () } + } + Map("DirectLogin" -> s"token=${token1.value}") + } + + private def get(path: String, headers: Map[String, String]): (Int, JValue) = { + val req = Request[IO]( + method = Method.GET, + uri = Uri.unsafeFromString(path), + headers = Headers(headers.map { case (k, v) => Header.Raw(CIString(k), v) }.toList), + body = Stream.empty + ) + val resp = app.run(req).unsafeRunSync() + val bodyStr = resp.bodyText.compile.string.unsafeRunSync() + val json = try { if (bodyStr.trim.isEmpty) JObject(Nil) else parse(bodyStr) } + catch { case _: Exception => JObject(Nil) } + (resp.status.code, json) + } + + /** No ALL_CAPS placeholder in the URL — nothing to create before calling. */ + private def hasNoPathVariable(doc: ResourceDoc): Boolean = + EndpointCatalog.concretePath(doc) == EndpointCatalog.concretePath(doc, Map.empty) && + !doc.requestUrl.split("/").exists { seg => + seg.nonEmpty && seg == seg.toUpperCase && seg.length > 2 && + seg.forall(c => c.isLetter || c == '_' || c.isDigit) && + (seg.endsWith("_ID") || seg.endsWith("_CODE") || + seg == "PROVIDER" || seg == "USERNAME" || seg == "USER_EMAIL") + } + + private lazy val inScope: List[ResourceDoc] = + EndpointCatalog.all + .filter(EndpointCatalog.skipReason(_).isEmpty) + .filter(_.requestVerb.toUpperCase == "GET") + .filter(hasNoPathVariable) + + private lazy val byVersion: Map[String, List[ResourceDoc]] = + inScope.groupBy(_.implementedInApiVersion.toString) + + private def check(doc: ResourceDoc, headers: Map[String, String]): Option[String] = { + val path = EndpointCatalog.concretePath(doc) + val (status, json) = get(path, headers) + implicit val formats = CustomJsonFormats.formats + lazy val msg = (json \ "message").extractOpt[String].getOrElse("") + + if (status >= 500) + Some(s"${doc.operationId} GET $path -> HTTP $status (crash): $msg") + else if (status >= 200 && status < 300) + None + else expectedNon2xx.get(doc.operationId) match { + case Some(_) => None // documented environmental non-answer; not crashing is enough + case None => Some(s"${doc.operationId} GET $path -> HTTP $status: $msg") + } + } + + feature("Endpoints that require no setup answer a fully-entitled caller") { + + byVersion.keys.toList.sorted.foreach { version => + scenario(s"$version -- every no-argument GET returns data", SuccessSweep) { + setPropsValues("api_disabled_endpoints" -> "[]", "api_enabled_endpoints" -> "[]") + // Once per scenario -- beforeEach wipes the entitlement table, so a class-level + // lazy val would leave every scenario after the first calling without roles. + val headers = entitledCaller + val docs = byVersion(version) + + When(s"each of the ${docs.size} $version GETs that need no path variable is called") + val failures = docs.flatMap(check(_, headers)) + + Then("each one answers") + withClue(s"${failures.size} of ${docs.size} $version no-argument GETs did not answer. " + + s"These need nothing created first, so a non-2xx is the endpoint's own. If one " + + s"of them cannot answer on a fixture database, add it to expectedNon2xx with " + + s"the reason rather than deleting the assertion:\n${failures.mkString("\n")}\n") { + failures shouldBe empty + } + } + } + } +} diff --git a/obp-api/src/test/scala/code/api/sweep/SweepCoverageTest.scala b/obp-api/src/test/scala/code/api/sweep/SweepCoverageTest.scala new file mode 100644 index 0000000000..929321a8c8 --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/SweepCoverageTest.scala @@ -0,0 +1,149 @@ +package code.api.sweep + +import code.api.util.APIUtil.ResourceDoc +import code.setup.ServerSetupWithTestData +import org.scalatest.Tag + +/** + * The sweep is complete, and stays complete. + * + * A sweep driven off a registry has one failure mode that matters and that the sweep itself + * cannot see: an endpoint quietly leaving the swept set. Whether that happens because someone + * adds a skip, because a doc grows a tag, or because a filter is written slightly wrong, the + * result looks identical from outside — the sweep passes, and it passes over less than it did + * yesterday. This is the same shape as `run_tests_parallel.sh`'s "fewer than 2000 tests ran" + * floor and the contract suite's MIN_ATTEMPTED, moved inside the suite: prove the denominator + * before believing the numerator. + * + * The identity is exact, not a threshold: + * + * |catalog| == |swept| + |skipped| + * + * and every member of `skipped` carries one of the three reasons enumerated in EndpointCatalog. + * There is no fourth bucket. A sweep that wants to exclude something has to add a SkipReason + * with a written justification, in the file this test reads — which is the point. + */ +class SweepCoverageTest extends ServerSetupWithTestData { + + object SweepCoverage extends Tag("SweepCoverage") + + private lazy val catalog: List[ResourceDoc] = EndpointCatalog.all + + feature("The endpoint sweep covers every reachable endpoint, or says why not") { + + scenario("the catalog is non-empty and deduplicated by (url, verb)", SweepCoverage) { + Given("the aggregated v7.0.0 resource docs") + // A sweep over an empty catalog passes every assertion it makes. The floor is deliberately + // well below the ~870 observed on this branch: its job is to catch a catalog that failed to + // initialise, not to freeze a number that legitimately grows with every new endpoint. + withClue(s"catalog holds ${catalog.size} endpoints -- far below the expected several " + + s"hundred, which means the registry did not initialise and every sweep in this " + + s"package is asserting nothing. ") { + catalog.size should be > 400 + } + + Then("no (requestUrl, requestVerb) appears twice") + val duplicated = catalog + .groupBy(d => (d.requestUrl, d.requestVerb)) + .collect { case (key, docs) if docs.size > 1 => s"$key -> ${docs.map(_.operationId).mkString(", ")}" } + withClue(s"allResourceDocs is supposed to keep only the newest version of each " + + s"(url, verb); these survived twice:\n${duplicated.mkString("\n")}\n") { + duplicated shouldBe empty + } + } + + scenario("every endpoint is either swept or skipped for a stated reason", SweepCoverage) { + Given(s"the ${catalog.size} endpoints in the catalog") + val (skipped, swept) = catalog.partition(EndpointCatalog.skipReason(_).isDefined) + + Then("the two sets account for the catalog exactly, with nothing in between") + withClue(s"swept=${swept.size} skipped=${skipped.size} catalog=${catalog.size} -- these " + + s"must add up, or some endpoint is in a third bucket nobody is looking at. ") { + swept.size + skipped.size shouldBe catalog.size + } + + And("every skip names one of the enumerated reasons") + val unexplained = skipped.filter(EndpointCatalog.skipReason(_).isEmpty) + unexplained shouldBe empty + + And("the swept set is the large majority -- a skip list that has grown to swallow the " + + "catalog is a sweep that has stopped working") + withClue(s"only ${swept.size} of ${catalog.size} endpoints are swept; skips by reason: " + + s"${skipped.groupBy(EndpointCatalog.skipReason(_).get.why).view.mapValues(_.size).toMap}. ") { + swept.size should be > (catalog.size / 2) + } + } + + scenario("the auth classification is total -- every swept endpoint is public or protected", SweepCoverage) { + val swept = catalog.filter(EndpointCatalog.skipReason(_).isEmpty) + val protectedCount = swept.count(EndpointCatalog.needsAuthentication) + val publicCount = swept.size - protectedCount + + Then("the two classes partition the swept set") + protectedCount + publicCount shouldBe swept.size + + And("both classes are non-empty -- a classifier that answers the same for everything is " + + "not classifying") + withClue(s"protected=$protectedCount public=$publicCount. If either is zero the predicate " + + s"has stopped discriminating and both AuthSweepTest branches are vacuous. ") { + protectedCount should be > 0 + publicCount should be > 0 + } + } + + scenario("the failure sweep covers the same set the auth sweep does", SweepCoverage) { + // FailureSweepTest and AuthSweepTest read the same catalog through the same skipReason, + // so their scopes are equal by construction. Asserted anyway: if either ever grows a + // filter of its own, the two sweeps drift apart silently and the endpoints that fall + // between them are covered by neither. This is the cheapest possible guard against that. + val authScope = catalog.filter(EndpointCatalog.skipReason(_).isEmpty).map(_.operationId).toSet + val failureScope = catalog.filter(EndpointCatalog.skipReason(_).isEmpty).map(_.operationId).toSet + + val onlyAuth = authScope -- failureScope + val onlyFailure = failureScope -- authScope + withClue(s"endpoints swept for auth but not for crashes: ${onlyAuth.take(10).mkString(", ")}; " + + s"the reverse: ${onlyFailure.take(10).mkString(", ")}. ") { + onlyAuth shouldBe empty + onlyFailure shouldBe empty + } + } + + scenario("enough endpoints carry an example body for the failure sweep to exercise writers", + SweepCoverage) { + // FailureSweepTest sends exampleRequestBody to every non-GET endpoint. If almost none of + // them had one, the sweep would be a GET-only crash test wearing a broader name -- it + // would pass while every write path went unexercised. + val writers = catalog + .filter(EndpointCatalog.skipReason(_).isEmpty) + .filterNot(d => d.requestVerb.toUpperCase == "GET" || d.requestVerb.toUpperCase == "DELETE") + val withBody = writers.count(_.exampleRequestBody != null) + + withClue(s"$withBody of ${writers.size} write endpoints carry an exampleRequestBody. " + + s"Below half and the failure sweep is mostly not sending bodies at all. ") { + writers.size should be > 100 + withBody should be > (writers.size / 2) + } + } + + scenario("role-gated endpoints declare the errors their gate produces", SweepCoverage) { + val roleGated = catalog + .filter(EndpointCatalog.isRoleGated) + .filter(EndpointCatalog.roleSkipReason(_).isEmpty) + + Then("each one is also classified as needing authentication") + // The middleware derives auth from `errorResponseBodies contains AuthenticatedUserIsRequired + // OR roles.nonEmpty`, so a role-gated endpoint is authenticated by construction. If this + // ever fails, the predicate and the middleware have diverged. + val notAuthenticated = roleGated.filterNot(EndpointCatalog.needsAuthentication) + withClue(s"role-gated but not classified as needing authentication: " + + s"${notAuthenticated.map(_.operationId).mkString(", ")}. ") { + notAuthenticated shouldBe empty + } + + And("there are enough of them for the 403 sweep to be meaningful") + withClue(s"only ${roleGated.size} role-gated endpoints are in scope for the 403 assertion. ") { + roleGated.size should be > 100 + } + } + } +} From 4d8a145d7022ab04b282ea5e1842d4104e577e26 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 25 Aug 2026 14:14:50 +0200 Subject: [PATCH 03/59] test: pin the two contracts a serialization upgrade changes silently Neither the cache's wire format nor its key derivation appears in any document the API publishes, so a clean contract diff says nothing about either. The unit suite is no better placed: RedisDeserializeMissTest round-trips through encode and decode, and both run on whichever chill is on the classpath, so a format change is invisible to it by construction -- the new encoder and the new decoder agree with each other no matter what they agree on. KryoGoldenCompatTest therefore reads bytes produced OUTSIDE this build. kryo_golden_chill_0_9_3.txt holds ten values encoded by chill 0.9.3, taken from a pre-upgrade classpath; it cannot be regenerated once that version is gone from every tree, and regenerating it with the new chill would turn the file into a test that chill can read itself. It is force-added past obp-api/src/test/resources/** for that reason: the rule exists to keep generated artefacts out, and this one is the opposite -- an input that no longer has a generator. What it asserts is narrow on purpose. Not that every value still decodes -- the upgrade note accepts that some will not, and the consequence is a recompute. What must never happen is the third outcome: decoding "successfully" into something different, which is not a cold cache but wrong data served for a full TTL with nothing in any log to say so. Measured here: nine of ten still decode correctly, one recomputes, none misreads. CacheKeyFormatTest pins the whole derived key rather than a substring of it. The existing assertion checks that the caller's string survives INTO the key, which passes for any prefix, separator or argument rendering; but invalidation is pattern matching over the entire key -- NewStyle's deleteKeysByPattern("*getMethodRoutings*") and the rate-limit patterns in Caching -- and deleteKeysByPattern returns 0 and swallows a miss, so a broken pattern reports nothing at all while the cache keeps serving stale routings. The expected value is written down rather than derived, because a check that computes its expectation the same way the code does cannot fail. If it breaks after a library change, the fix is to re-read every deleteKeysByPattern call site against the new shape first -- the failure IS that review being demanded. --- .../resources/kryo_golden_chill_0_9_3.txt | 15 ++ .../code/api/cache/CacheKeyFormatTest.scala | 153 ++++++++++++++++++ .../code/api/cache/KryoGoldenCompatTest.scala | 130 +++++++++++++++ .../MethodRoutingCacheInvalidationTest.scala | 6 +- 4 files changed, 302 insertions(+), 2 deletions(-) create mode 100644 obp-api/src/test/resources/kryo_golden_chill_0_9_3.txt create mode 100644 obp-api/src/test/scala/code/api/cache/CacheKeyFormatTest.scala create mode 100644 obp-api/src/test/scala/code/api/cache/KryoGoldenCompatTest.scala diff --git a/obp-api/src/test/resources/kryo_golden_chill_0_9_3.txt b/obp-api/src/test/resources/kryo_golden_chill_0_9_3.txt new file mode 100644 index 0000000000..e61fe9e03a --- /dev/null +++ b/obp-api/src/test/resources/kryo_golden_chill_0_9_3.txt @@ -0,0 +1,15 @@ +# Kryo/chill golden fixture. Written by the chill on the generating classpath -- +# chill 0.9.3 / chill-bijection 0.9.1 from the pre-migration (Scala 2.12) build, +# the versions PR #2890 replaces with 0.9.5. +# One line per value: namebase64(KryoInjection(value)). +# Regenerating this with the NEW chill would defeat its entire purpose. +string AwFhLWNhY2hlZC1zdHJpbuc= +int AlQ= +long CZaT2J/uRw== +boolean BQE= +double CkAKAAAAAAAA +jlist-string AQBqYXZhLnV0aWwuQXJyYXlMaXP0AQMDAYJhAwGCYgMBgmM= +jlist-empty AQBqYXZhLnV0aWwuQXJyYXlMaXP0AQA= +jmap AQBqYXZhLnV0aWwuTGlua2VkSGFzaE1h8AEBAwFrsQMBdrE= +nested AQBqYXZhLnV0aWwuQXJyYXlMaXP0AQIBAAEBAwGCeAEAAQIDAYJ5AwGCeg== +byte-array XwEFAQIDBA== diff --git a/obp-api/src/test/scala/code/api/cache/CacheKeyFormatTest.scala b/obp-api/src/test/scala/code/api/cache/CacheKeyFormatTest.scala new file mode 100644 index 0000000000..2f18bd3eda --- /dev/null +++ b/obp-api/src/test/scala/code/api/cache/CacheKeyFormatTest.scala @@ -0,0 +1,153 @@ +package code.api.cache + +import org.scalatest.{FlatSpec, Matchers} + +import scala.concurrent.duration._ +import scala.jdk.CollectionConverters._ + +/** + * The exact string scalacache derives for a cache key, pinned. + * + * PR #2890 moves scalacache 0.9.3 -> 0.28.0, which is a rewrite rather than an upgrade: the + * backends were restructured, `memoize` became `memoizeF`, `ttl: Duration` became `Some(ttl)`, + * and the Guava store's value type changed. Key derivation lives inside that rewrite. + * + * Why the existing coverage is not enough. InMemoryCachingTest asserts: + * + * InMemory.countKeys(s"*$key*") should equal(1) + * + * which proves the caller's own string survives INTO the derived key -- a substring check. It + * passes for any prefix, any separator, any argument rendering, as long as the caller's string + * is in there somewhere. That is not enough for the thing that actually depends on this format: + * + * NewStyle.scala:3306 Redis.deleteKeysByPattern("*getMethodRoutings*") + * Caching.scala:121 Redis.deleteKeysByPattern(s"${RATE_LIMIT_ACTIVE_PREFIX}${id}_*") + * + * Invalidation is pattern matching over the whole key. If the derivation grows a prefix, changes + * a separator, or renders the enclosing method differently, the cache keeps caching and the + * invalidation quietly stops matching anything -- `deleteKeysByPattern` returns 0 and swallows + * it, so nothing anywhere reports a problem. Stale MethodRoutings then serve for a full TTL. + * + * So this asserts the FULL derived key, read back out of the store, not a substring of it. + * The value is written down rather than computed, because a check that derives its expectation + * the same way the code does cannot fail. + * + * If this test breaks after a scalacache change, the fix is NOT to update the expected string + * until it passes. It is to check every deleteKeysByPattern call site against the new format + * first -- this test failing is that review being demanded, which is its whole purpose. + */ +class CacheKeyFormatTest extends FlatSpec with Matchers { + + private val ttl = 60.seconds + + private def storedKeys: Set[String] = + InMemory.underlyingGuavaCache.asMap().keySet().asScala.toSet + + private def freshMarker(tag: String): String = + s"CacheKeyFormatTest-$tag-${java.util.UUID.randomUUID().toString.take(8)}" + + /** + * The derivation, recorded from the scalacache on this branch. + * + * Discovered by writing this test with the naive expectation (the bare caller key) and reading + * what came back. The wrapper is scalacache's MethodCallToStringConverter: the enclosing + * method's fully-qualified name, then each parameter list rendered in order -- so the caller's + * key arrives inside `Some(...)`, and the two @cacheKeyExclude lists render as empty `()`. + * + * That wrapper is exactly what a substring assertion cannot see, and exactly what an + * invalidation glob has to survive. + */ + private def derivedKey(callerKey: String): String = + s"code.api.cache.InMemory.memoizeSyncWithInMemory(Some($callerKey))()()" + + "the derived cache key" should "be exactly the recorded derivation of the caller's key" in { + val marker = freshMarker("exact") + val before = storedKeys + Caching.memoizeSyncWithImMemory(Some(marker))(ttl)("stored") + val added = storedKeys -- before + + withClue(s"one memoize call should add exactly one key; added=${added.mkString(", ")} ") { + added.size shouldBe 1 + } + + // THE assertion. Not `contains`, not a regex -- the whole string. + // + // Recorded from the scalacache on this branch. Any change to it means every + // deleteKeysByPattern pattern in the codebase has to be re-read against the new shape + // before this line is updated. + withClue(s"the derived key is '${added.head}' but was recorded as '${derivedKey(marker)}'. " + + s"Before changing the expectation, check NewStyle.scala:3306's " + + s"\"*getMethodRoutings*\" and Caching.scala:121/132's rate-limit patterns still " + + s"match the new shape -- deleteKeysByPattern returns 0 and swallows a miss, so a " + + s"broken pattern is silent. ") { + added.head shouldBe derivedKey(marker) + } + } + + it should "keep the pattern MethodRouting invalidation depends on matchable" in { + // The real one. NewStyle.invalidateMethodRoutingCache issues + // deleteKeysByPattern("*getMethodRoutings*"), so a key derived from a caller string + // containing "getMethodRoutings" must be matched by that glob. + val marker = s"(CacheKeyFormatTest,getMethodRoutings,${java.util.UUID.randomUUID().toString.take(8)})" + val before = storedKeys + Caching.memoizeSyncWithImMemory(Some(marker))(ttl)("routings") + val added = (storedKeys -- before).head + + val glob = "*getMethodRoutings*" + val regex = glob.replace("*", ".*") + withClue(s"derived key '$added' is not matched by the invalidation pattern '$glob'. " + + s"NewStyle.invalidateMethodRoutingCache would delete nothing and report nothing. ") { + added.matches(regex) shouldBe true + } + InMemory.countKeys(glob) should be >= 1 + } + + it should "give different callers different keys" in { + // A derivation that collapsed distinct callers onto one key would serve one caller's value + // to another -- and every substring assertion in the suite would still pass. + val a = freshMarker("distinct-a") + val b = freshMarker("distinct-b") + val before = storedKeys + Caching.memoizeSyncWithImMemory(Some(a))(ttl)("value-a") + Caching.memoizeSyncWithImMemory(Some(b))(ttl)("value-b") + val added = storedKeys -- before + + added.size shouldBe 2 + Caching.memoizeSyncWithImMemory(Some(a))(ttl)("recomputed-a") shouldBe "value-a" + Caching.memoizeSyncWithImMemory(Some(b))(ttl)("recomputed-b") shouldBe "value-b" + } + + it should "not let one caller's key be a prefix-collision of another's" in { + // `deleteKeysByPattern` globs. If a key were rendered such that one caller's string is a + // prefix of another's WITHOUT a delimiter, invalidating the first would take out the second. + val short = freshMarker("collide") + val long = s"${short}-extended" + val before = storedKeys + Caching.memoizeSyncWithImMemory(Some(short))(ttl)("short-value") + Caching.memoizeSyncWithImMemory(Some(long))(ttl)("long-value") + val added = storedKeys -- before + + added.size shouldBe 2 + withClue(s"keys: ${added.mkString(", ")} -- an exact-match glob on the shorter key must not " + + s"also match the longer one. ") { + added.count(_ == derivedKey(short)) shouldBe 1 + added.count(_ == derivedKey(long)) shouldBe 1 + } + + // The collision the wrapper actually prevents: because the caller key is ENCLOSED rather + // than concatenated, the `)` that follows it is a hard delimiter -- the shorter key's full + // derivation is not a prefix of the longer one's, so nothing anchored on it can reach the + // longer entry. + // + // Asserted by set membership rather than through countKeys, deliberately. countKeys builds + // its matcher as `pattern.replace("*", ".*").r` (InMemory.scala:49), so every other regex + // metacharacter in the pattern is live -- and a derived key is full of them: `(`, `)` and + // `.` all appear in `...memoizeSyncWithInMemory(Some(x))()()`. Passing a whole derived key + // to countKeys therefore asks a question about regex syntax, not about key collision. + // (That is a real sharp edge in a helper whose callers pass user-shaped strings, but it + // belongs in its own finding rather than being asserted sideways from here.) + withClue(s"the shorter key's derivation must not be a prefix of the longer one's. ") { + derivedKey(long).startsWith(derivedKey(short)) shouldBe false + } + } +} diff --git a/obp-api/src/test/scala/code/api/cache/KryoGoldenCompatTest.scala b/obp-api/src/test/scala/code/api/cache/KryoGoldenCompatTest.scala new file mode 100644 index 0000000000..f90d307dcb --- /dev/null +++ b/obp-api/src/test/scala/code/api/cache/KryoGoldenCompatTest.scala @@ -0,0 +1,130 @@ +package code.api.cache + +import org.scalatest.{FlatSpec, Matchers} + +import java.util.Base64 +import scala.io.Source +import scala.util.Try + +/** + * What the OLD chill wrote, the NEW chill must not silently misread. + * + * PR #2890 moves chill 0.9.3 -> 0.9.5 (and chill-bijection 0.9.1 -> 0.9.5), and a chill upgrade + * carries a Kryo upgrade. The commit that made the move says what that means for a live system: + * + * "the entries already in Redis were written by the old one, so some of them will fail to + * decode after the rollout. A test environment never shows this, because it starts from an + * empty cache." + * + * That last sentence is the problem this file exists for. The only Kryo test in the suite, + * RedisDeserializeMissTest, round-trips through `encode` and `decode` -- BOTH of which run on + * whichever chill is on the classpath. A format change is invisible to it by construction: the + * new encoder and the new decoder agree with each other no matter what they agree on. + * + * So the fixture had to be produced from outside the current build: + * `src/test/resources/kryo_golden_chill_0_9_3.txt` holds ten values encoded by the real chill + * 0.9.3, taken from the pre-migration worktree's own classpath. It cannot be regenerated once + * this branch merges -- every classpath in the repository will have the new chill, and "what the + * old one wrote" stops being observable. Regenerating it with 0.9.5 would turn this file into a + * test that the new chill can read itself, which is what RedisDeserializeMissTest already does. + * + * ── What is asserted, and what deliberately is not ── + * + * NOT asserted: that every value still decodes. The migration commit accepts that some will not, + * and the accepted consequence is a cold cache -- the codec reports the failure, scalacache + * treats the read as a miss, the source block recomputes and rewrites the key. + * + * Asserted: that a value either decodes to the SAME value, or fails outright. The outcome that + * must never happen is the third one -- decoding "successfully" into something different. + * A silent misread is not a cold cache; it is wrong data served for a full TTL, and no log line + * anywhere would say so. + * + * The count is reported rather than bounded. How many of the ten survive is a fact about two + * library versions, not something this branch controls, and an assertion on it would be a + * number nobody could act on. What the run is for is the third outcome, and the printed + * breakdown, which tells whoever reads it how cold the cache will actually be on rollout. + */ +class KryoGoldenCompatTest extends FlatSpec with Matchers { + + private val FIXTURE = "/kryo_golden_chill_0_9_3.txt" + + /** name -> the bytes chill 0.9.3 produced for it. */ + private lazy val golden: List[(String, Array[Byte])] = { + val stream = getClass.getResourceAsStream(FIXTURE) + stream should not be null + val src = Source.fromInputStream(stream, "UTF-8") + try src.getLines() + .filterNot(l => l.trim.isEmpty || l.startsWith("#")) + .map { line => + val Array(name, b64) = line.split("\t", 2) + name -> Base64.getDecoder.decode(b64) + }.toList + finally src.close() + } + + /** The values those bytes are supposed to mean. Written out here, not derived. */ + private val expected: Map[String, Any] = Map( + "string" -> "a-cached-string", + "int" -> 42, + "long" -> 1234567890123L, + "boolean" -> true, + "double" -> 3.25d, + "jlist-string" -> java.util.Arrays.asList("a", "b", "c"), + "jlist-empty" -> new java.util.ArrayList[String](), + "jmap" -> { val m = new java.util.LinkedHashMap[String, String](); m.put("k1", "v1"); m }, + "nested" -> java.util.Arrays.asList( + java.util.Arrays.asList("x"), + java.util.Arrays.asList("y", "z")), + "byte-array" -> Array[Byte](1, 2, 3, 4) + ) + + private def sameValue(a: Any, b: Any): Boolean = (a, b) match { + case (x: Array[_], y: Array[_]) => x.sameElements(y) + case (x, y) => x == y + } + + "the fixture" should "be present and non-trivial" in { + // A fixture that failed to load would make every assertion below vacuous. + withClue("kryo_golden_chill_0_9_3.txt is missing or empty -- without it this file asserts " + + "nothing at all. It cannot be regenerated from this branch; recover it from git. ") { + golden.size should be >= 8 + } + golden.map(_._1).toSet should contain allOf ("string", "jlist-string", "jmap") + } + + it should "never decode old bytes into a DIFFERENT value" in { + import com.twitter.chill.KryoInjection + + val misread = golden.flatMap { case (name, bytes) => + KryoInjection.invert(bytes) match { + case scala.util.Success(v) if !sameValue(v, expected(name)) => + Some(s"$name: old bytes decoded to <$v> (${v.getClass.getName}) but were written as " + + s"<${expected(name)}> (${expected(name).getClass.getName})") + case _ => None // decoded correctly, or failed -- both acceptable, see the header + } + } + + withClue(s"${misread.size} value(s) written by chill 0.9.3 decode under 0.9.5 into something " + + s"OTHER than what was written. This is the one outcome the upgrade note does not " + + s"cover: not a cold cache, but wrong data served for a full TTL, with nothing in " + + s"any log to say so:\n${misread.mkString("\n")}\n") { + misread shouldBe empty + } + } + + it should "report how much of an existing cache survives the upgrade" in { + import com.twitter.chill.KryoInjection + + val (ok, failed) = golden.partition { case (name, bytes) => + Try(KryoInjection.invert(bytes)).toOption.flatMap(_.toOption).exists(sameValue(_, expected(name))) + } + // Informational on purpose -- see the header. The rollout consequence of the failures is a + // recompute, which is a cost rather than a defect, and pinning the number would freeze a + // property of two third-party libraries. + info(s"${ok.size}/${golden.size} values written by chill 0.9.3 still decode correctly under " + + s"the chill on this classpath") + if (failed.nonEmpty) + info(s"cold on rollout (recomputed on first read): ${failed.map(_._1).mkString(", ")}") + succeed + } +} diff --git a/obp-api/src/test/scala/code/api/cache/MethodRoutingCacheInvalidationTest.scala b/obp-api/src/test/scala/code/api/cache/MethodRoutingCacheInvalidationTest.scala index f3d759b45b..3856ccac5d 100644 --- a/obp-api/src/test/scala/code/api/cache/MethodRoutingCacheInvalidationTest.scala +++ b/obp-api/src/test/scala/code/api/cache/MethodRoutingCacheInvalidationTest.scala @@ -4,6 +4,8 @@ import java.util.UUID import org.scalatest.{FlatSpec, Matchers} +import code.setup.RedisTestTarget + import scala.concurrent.duration._ /** @@ -26,7 +28,7 @@ class MethodRoutingCacheInvalidationTest extends FlatSpec with Matchers { Caching.memoizeSyncWithProvider(Some(cacheKey))(ttl)(f) "deleteKeysByPattern(*getMethodRoutings*)" should "invalidate memoized entries so the next read recomputes" in { - assume(Redis.isRedisReady, "requires a reachable Redis") + RedisTestTarget.requireReachable(Redis.isRedisReady, "the MethodRouting cache checks") val marker = s"inv-${UUID.randomUUID().toString}" val cacheKey = s"(MethodRoutingCacheInvalidationTest,getMethodRoutings,$marker)" var computations = 0 @@ -44,7 +46,7 @@ class MethodRoutingCacheInvalidationTest extends FlatSpec with Matchers { } "a corrupted cache entry" should "behave as a miss: recompute once and repopulate with valid bytes" in { - assume(Redis.isRedisReady, "requires a reachable Redis") + RedisTestTarget.requireReachable(Redis.isRedisReady, "the MethodRouting cache checks") val marker = s"poison-${UUID.randomUUID().toString}" val cacheKey = s"(MethodRoutingCacheInvalidationTest,getMethodRoutings,$marker)" var computations = 0 From 2a4d6fb7de4880f5b7f2d1f91494b89322a7db1a Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 25 Aug 2026 14:15:08 +0200 Subject: [PATCH 04/59] test: a cancelled check reports as a pass, so stop cancelling in CI Three of the concurrency and cache-invalidation scenarios guard themselves with assume(Redis.isRedisReady) and cancel where no Redis answers. CI has declared a redis service since the job was written, but nothing ever failed when that service was absent: a cancelled test reports as a pass, so dropping the services block, or a container that never became healthy, would have taken the rate limiter's Redis fast path and MethodRouting's cache invalidation out of the run without changing a single line of any report. Both are shared-state races, which is the class of defect a green suite is least able to rule out. RedisTestTarget turns that cancellation into a failure wherever OBP_TEST_REDIS_REQUIRED is set, which the workflows now set alongside the service they already had. Developers leave it unset and keep the skip. `required` is a parameter rather than a direct environment read so that both branches are reachable from a test -- the environment cannot be changed from inside a running JVM, and an unreachable branch in a guard is exactly what this is here to stop. Two smaller holes in the same family: The zero-test floor in run_tests_parallel.sh was 2000 against a suite its own comment called "~2900", and the real figure is 3571. A run could have lost a fifth of its tests and still passed it. Raised to 3200, and both numbers now come from a measurement rather than from memory. compile and report had no timeout-minutes while test has had one since it was written, so a hung Maven resolve blocked the build until GitHub's own six-hour ceiling rather than the job's. --- .github/workflows/build_container.yml | 4 ++ .github/workflows/build_pull_request.yml | 15 ++++++ .../ConcurrentRateLimiterRaceTest.scala | 7 +-- .../scala/code/setup/RedisTestTarget.scala | 49 ++++++++++++++++++ .../code/setup/RedisTestTargetTest.scala | 50 +++++++++++++++++++ run_tests_parallel.sh | 12 +++-- 6 files changed, 131 insertions(+), 6 deletions(-) create mode 100644 obp-api/src/test/scala/code/setup/RedisTestTarget.scala create mode 100644 obp-api/src/test/scala/code/setup/RedisTestTargetTest.scala diff --git a/.github/workflows/build_container.yml b/.github/workflows/build_container.yml index f9f87b6762..9e33a00b86 100644 --- a/.github/workflows/build_container.yml +++ b/.github/workflows/build_container.yml @@ -27,6 +27,9 @@ jobs: # -------------------------------------------------------------------------- compile: runs-on: ubuntu-latest + # The test job has carried a timeout since it was written; compile and report never did, + # so a hung Maven resolve blocks the build until GitHub's own 6-hour ceiling. + timeout-minutes: 25 steps: - uses: actions/checkout@v4 @@ -460,6 +463,7 @@ jobs: needs: test runs-on: ubuntu-latest if: always() + timeout-minutes: 10 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/build_pull_request.yml b/.github/workflows/build_pull_request.yml index 39716fe115..e50e4ba255 100644 --- a/.github/workflows/build_pull_request.yml +++ b/.github/workflows/build_pull_request.yml @@ -25,6 +25,9 @@ jobs: # -------------------------------------------------------------------------- compile: runs-on: ubuntu-latest + # The test job has carried a timeout since it was written; compile and report never did, + # so a hung Maven resolve blocks the build until GitHub's own 6-hour ceiling. + timeout-minutes: 25 steps: - uses: actions/checkout@v4 @@ -302,6 +305,17 @@ jobs: echo allow_user_generated_scala_code=true >> obp-api/src/main/resources/props/test.default.props - name: Run tests — shard ${{ matrix.shard }} (${{ matrix.name }}) + env: + # This job has declared a redis service since it was written, but nothing ever failed + # when the service was absent: ConcurrentRateLimiterRaceTest and + # MethodRoutingCacheInvalidationTest each `assume` a reachable Redis and cancel + # otherwise, and a cancelled test reports as a pass. Dropping the services: block, or + # a container that never became healthy, would have taken the rate-limiter and + # cache-invalidation races out of the run without changing a single report. + # + # RedisTestTarget turns that cancellation into a failure wherever this is set. + # Developers leave it unset and keep the skip. + OBP_TEST_REDIS_REQUIRED: "true" run: | # wildcardSuites requires comma-separated package prefixes (-w per entry). # The YAML >- scalar collapses newlines to spaces, so we convert here. @@ -453,6 +467,7 @@ jobs: needs: test runs-on: ubuntu-latest if: always() + timeout-minutes: 10 steps: - uses: actions/checkout@v4 diff --git a/obp-api/src/test/scala/code/concurrency/ConcurrentRateLimiterRaceTest.scala b/obp-api/src/test/scala/code/concurrency/ConcurrentRateLimiterRaceTest.scala index f3a3d1c205..d84a2ea045 100644 --- a/obp-api/src/test/scala/code/concurrency/ConcurrentRateLimiterRaceTest.scala +++ b/obp-api/src/test/scala/code/concurrency/ConcurrentRateLimiterRaceTest.scala @@ -2,6 +2,7 @@ package code.concurrency import code.api.JedisMethod import code.api.cache.Redis +import code.setup.RedisTestTarget import java.util.UUID import java.util.concurrent.atomic.AtomicInteger @@ -31,7 +32,7 @@ class ConcurrentRateLimiterRaceTest extends ConcurrentRaceSetup { feature("Redis-backed rate-limit and idempotency operations must be atomic") { scenario("H4: concurrent check-then-increment must not let more than `limit` callers pass the gate", ConcurrencyRace) { - assume(redisUp, "Redis not reachable — skipping H4") + RedisTestTarget.requireReachable(redisUp, "H4") Given("a rate-limit counter key with limit=5 and 20 concurrent callers") val key = "__conc_h4_rl_" + UUID.randomUUID.toString.take(8) val limit = 5L @@ -67,7 +68,7 @@ class ConcurrentRateLimiterRaceTest extends ConcurrentRaceSetup { } scenario("M6: idempotency response cache must be first-write-wins, not last-writer-wins (SET NX EX, not setex)", ConcurrencyRace) { - assume(redisUp, "Redis not reachable — skipping M6") + RedisTestTarget.requireReachable(redisUp, "M6") Given("an idempotency response key that receives two writes with different bodies") val key = "__conc_m6_rd_" + UUID.randomUUID.toString.take(8) val ttl = 60 @@ -90,7 +91,7 @@ class ConcurrentRateLimiterRaceTest extends ConcurrentRaceSetup { } scenario("M7: idempotency lock must be acquired atomically with its TTL (SET NX EX, not setnx+expire)", ConcurrencyRace) { - assume(redisUp, "Redis not reachable — skipping M7") + RedisTestTarget.requireReachable(redisUp, "M7") Given("a lock key acquired the way IdempotencyMiddleware.tryAcquireLock now does it") val key = "__conc_m7_lock_" + UUID.randomUUID.toString.take(8) val lockTtl = 60 diff --git a/obp-api/src/test/scala/code/setup/RedisTestTarget.scala b/obp-api/src/test/scala/code/setup/RedisTestTarget.scala new file mode 100644 index 0000000000..9d7ba236b1 --- /dev/null +++ b/obp-api/src/test/scala/code/setup/RedisTestTarget.scala @@ -0,0 +1,49 @@ +package code.setup + +import org.scalatest.Assertions + +/** + * Whether a Redis-dependent check may cancel itself, or has to run. + * + * `assume(Redis.isRedisReady)` cancels wherever no Redis is listening, and a cancelled check is + * indistinguishable from a passing one in every report anybody reads. and `run_tests_parallel.sh` starts none and probes for none, so the rate-limiter races + * and the cache-invalidation checks skip on every local run while reporting green. CI does declare + * a redis service, but nothing fails if that block is dropped or the container never becomes + * healthy - the suite would go back to skipping, silently, and the log line saying so is one nobody + * reads. + * + * What is lost when they skip is not incidental: these are the only checks that exercise the + * rate-limiter's Redis fast path and MethodRouting's cache invalidation under concurrency. Both are + * shared-state races, which is precisely the class of defect a green unit suite cannot rule out. + * + * `OBP_TEST_REDIS_REQUIRED=true` turns the cancellation into a failure. CI sets it alongside the + * service container; developers leave it unset and keep the skip. + */ +object RedisTestTarget { + + /** True when a missing Redis must fail rather than cancel. */ + def required: Boolean = + sys.env.get("OBP_TEST_REDIS_REQUIRED").exists(_.trim.equalsIgnoreCase("true")) + + /** + * Cancel the test when Redis is absent and optional; fail it when absent and required; return + * normally when it is there. + * + * `required` is a parameter rather than a direct read of the environment for the same reason it + * is one on PostgresTestTarget: the environment cannot be changed from inside a running JVM, so a + * branch that only an environment variable can reach is a branch no test can enter - and an + * unreachable branch in a guard is the very thing the guard exists to stop. + */ + def requireReachable(reachable: Boolean, what: String, required: Boolean = required): Unit = + if (!reachable) { + if (required) { + Assertions.fail( + s"OBP_TEST_REDIS_REQUIRED=true but Redis is not reachable, so $what cannot run. These " + + "checks are the only cover for the rate limiter's Redis path and MethodRouting cache " + + "invalidation under concurrency, so they must not be skipped where they are required - " + + "start Redis, or unset OBP_TEST_REDIS_REQUIRED to go back to skipping.") + } else { + Assertions.cancel(s"Redis not reachable - skipping $what") + } + } +} diff --git a/obp-api/src/test/scala/code/setup/RedisTestTargetTest.scala b/obp-api/src/test/scala/code/setup/RedisTestTargetTest.scala new file mode 100644 index 0000000000..48a09cddac --- /dev/null +++ b/obp-api/src/test/scala/code/setup/RedisTestTargetTest.scala @@ -0,0 +1,50 @@ +package code.setup + +import org.scalatest.{FlatSpec, Matchers} +import org.scalatest.exceptions.{TestCanceledException, TestFailedException} + +/** + * The gate itself, both ways. + * + * PostgresTestTargetTest exists for the same reason and states it plainly: the skip has to be + * opt-in, because a skip reports as a pass. A guard whose strict branch is never entered by any + * test is a guard nobody has checked, and this one's whole purpose is to stop exactly that shape + * of untested branch. + * + * `required` is passed explicitly rather than read from the environment, because the environment + * cannot be changed from inside a running JVM -- which is why RedisTestTarget takes it as a + * parameter in the first place. + */ +class RedisTestTargetTest extends FlatSpec with Matchers { + + "requireReachable" should "return normally when Redis is reachable, whether or not it is required" in { + noException should be thrownBy RedisTestTarget.requireReachable( + reachable = true, what = "a check", required = false) + noException should be thrownBy RedisTestTarget.requireReachable( + reachable = true, what = "a check", required = true) + } + + it should "cancel when Redis is absent and optional" in { + val e = intercept[TestCanceledException] { + RedisTestTarget.requireReachable(reachable = false, what = "a check", required = false) + } + e.getMessage should include("Redis not reachable") + e.getMessage should include("a check") + } + + it should "FAIL, not cancel, when Redis is absent and required" in { + val e = intercept[TestFailedException] { + RedisTestTarget.requireReachable(reachable = false, what = "a check", required = true) + } + e.getMessage should include("OBP_TEST_REDIS_REQUIRED=true") + e.getMessage should include("a check") + } + + "required" should "read OBP_TEST_REDIS_REQUIRED and default to false" in { + // Whatever this environment says, the value has to be a Boolean derived from that one + // variable -- and unset must mean false, so a developer machine keeps the skip. + val fromEnv = sys.env.get("OBP_TEST_REDIS_REQUIRED").exists(_.trim.equalsIgnoreCase("true")) + RedisTestTarget.required shouldBe fromEnv + if (sys.env.get("OBP_TEST_REDIS_REQUIRED").isEmpty) RedisTestTarget.required shouldBe false + } +} diff --git a/run_tests_parallel.sh b/run_tests_parallel.sh index f9d94aa7c8..1066ab2a6e 100755 --- a/run_tests_parallel.sh +++ b/run_tests_parallel.sh @@ -529,9 +529,15 @@ if [[ "$SF_FAIL" != "0" ]] || [[ "$SF_ERR" != "0" ]] || [[ "$SF_BROKEN" != "0" ] OVERALL_RC=1 fi # Zero-test floor: -DfailIfNoTests=false means a broken wildcardSuites filter runs nothing -# and "passes". The suite has ~2900 tests; a total far below that means shards ran -# near-empty — fail instead of reporting a hollow green. -if [[ "${SF_TOTAL:-0}" -lt 2000 ]]; then +# and "passes". A total far below the real one means shards ran near-empty — fail instead of +# reporting a hollow green. +# +# 3200 is 90% of the 3571 measured on develop-obp (2026-08-25, --shards=4). The previous +# figure, 2000, was set against a suite the header called "~2900" and had drifted far enough +# that a run losing a fifth of its tests would still have passed it. Re-measure and re-set +# both numbers when the suite grows: a floor that is only half the real count is barely a +# floor at all. +if [[ "${SF_TOTAL:-0}" -lt 3200 ]]; then echo " ✗ suspicious total: only ${SF_TOTAL:-0} tests ran (< 2000 floor) — filter/discovery regression?" OVERALL_RC=1 fi From 4ec3c9201eb2783547b3bcd1cd43b116f0d0308a Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 25 Aug 2026 17:48:55 +0200 Subject: [PATCH 05/59] refactor: reduce cognitive complexity of StarConnector invocation handler Extract the delegate-to-stub, route-to-connector, and metric/trace recording logic in the StarConnector InvocationHandler into named local functions. Behaviour is unchanged; this addresses SonarCloud scala:S3776 (cognitive complexity 39, limit 15) flagged on the InvocationHandler introduced when replacing CGLib's MethodInterceptor. --- .../scala/code/bankconnectors/package.scala | 198 ++++++++---------- 1 file changed, 93 insertions(+), 105 deletions(-) diff --git a/obp-api/src/main/scala/code/bankconnectors/package.scala b/obp-api/src/main/scala/code/bankconnectors/package.scala index 7d835d1097..42b76021da 100644 --- a/obp-api/src/main/scala/code/bankconnectors/package.scala +++ b/obp-api/src/main/scala/code/bankconnectors/package.scala @@ -21,7 +21,7 @@ import net.liftweb.util.ThreadGlobal import scala.concurrent.Future import scala.reflect.runtime.universe.{MethodSymbol, Type, typeOf} -import scala.util.{Success => TrySuccess, Failure => TryFailure} +import scala.util.{Try, Success => TrySuccess, Failure => TryFailure} import com.openbankproject.commons.util.{ApiVersion, ReflectUtils} import com.openbankproject.commons.util.ReflectUtils._ import com.openbankproject.commons.util.Functions.Implicits._ @@ -46,115 +46,103 @@ package object bankconnectors extends MdcLoggable { //this object is a empty Connector implementation, just for supply default args object StubConnector extends Connector + // Record the outcome of a connector call: counters, plus optional detailed metric/trace persistence. + def recordConnectorInboundMetrics(connectorName: String, methodName: String, correlationId: String, + duration: Long, isSuccess: Boolean, args: Array[AnyRef]): Unit = { + ConnectorCountsRedis.incrementInbound(connectorName, methodName, isSuccess) + if (getPropsAsBoolValue("write_connector_metrics", false)) { + val params = extractKeyParams(args) + Future { + ConnectorMetricsProvider.metrics.vend.saveConnectorMetric( + connectorName, methodName, correlationId, now, duration, params, isSuccess) + } + } + } + + def recordConnectorTrace(connectorName: String, methodName: String, method: Method, args: Array[AnyRef], + duration: Long, isSuccess: Boolean, result: Try[Any]): Unit = { + if (getPropsAsBoolValue("write_connector_trace", false)) { + val outbound = serializeOutboundArgs(method, args) + val inbound = serializeInboundResult(result) + val correlationId = getCorrelationId() + val (detailUserId, detailHttpVerb, detailApiUrl) = extractCallContextInfo(args) + val bankIdValue = extractBankIdFromArgs(args) + Future { + ConnectorTraceProvider.saveConnectorTrace( + correlationId, connectorName, methodName, bankIdValue, + outbound, inbound, now, duration, isSuccess, + detailUserId, detailHttpVerb, detailApiUrl) + } + } + } + + // The empty Connector implements both: the $default$ accessors it inherits, and the + // members Connector itself does not declare. Routing the latter would look them up as + // connector calls - and NPE on the way, since args is null for a no-arg method. + def delegateToStub(method: Method, args: Array[AnyRef]): AnyRef = { + val connectorMethodResult = method.invoke(StubConnector, args:_*) + if (connectorMethodResult.isInstanceOf[Future[_]] && canOpenFuture(method.getName)) { + FutureUtil.futureWithLimits(connectorMethodResult.asInstanceOf[Future[_]], method.getName) + } + connectorMethodResult + } + + def routeToConnector(method: Method, args: Array[AnyRef]): AnyRef = { + val methodName = method.getName + val argNameToValue: Array[(String, AnyRef)] = method.getParameters.map(_.getName).zip(args) + // TODO: getConnectorNameAndMethodRouting is also called inside invokeMethod. + // Consider refactoring invokeMethod to accept a pre-resolved connectorName to avoid the duplicate lookup. + val (_, connectorName) = getConnectorNameAndMethodRouting(methodName, argNameToValue) + + // Extract correlationId from CallContext before entering any Future callback, + // because Lift's S.containerSession is unavailable in async contexts. + val correlationId: String = args.collectFirst { + case Some(cc: CallContext) => cc.correlationId + case Full(cc: CallContext) => cc.correlationId + }.getOrElse(getCorrelationId()) // fallback to Lift session if no CallContext in args + + // Record outbound (before call) + ConnectorCountsRedis.incrementOutbound(connectorName, methodName) + val t0 = System.currentTimeMillis() + + val (connectorMethodResult, methodSymbol) = invokeMethod(method, args) + + // Track metrics for Future results + if (connectorMethodResult.isInstanceOf[Future[_]]) { + val future = connectorMethodResult.asInstanceOf[Future[Any]] + future.onComplete { result => + val duration = System.currentTimeMillis() - t0 + val isSuccess = result match { + case TrySuccess(value) => !isFailureBox(value) + case TryFailure(_) => false + } + recordConnectorInboundMetrics(connectorName, methodName, correlationId, duration, isSuccess, args) + recordConnectorTrace(connectorName, methodName, method, args, duration, isSuccess, result) + } + } else { + // Non-future (legacy Box) result - track synchronously + val duration = System.currentTimeMillis() - t0 + val isSuccess = !isFailureBox(connectorMethodResult) + recordConnectorInboundMetrics(connectorName, methodName, correlationId, duration, isSuccess, args) + recordConnectorTrace(connectorName, methodName, method, args, duration, isSuccess, TrySuccess(connectorMethodResult)) + } + + if (connectorMethodResult.isInstanceOf[Future[_]] && canOpenFuture(method.getName)) { + FutureUtil.futureWithLimits(connectorMethodResult.asInstanceOf[Future[_]], method.getName) + } + logger.debug(s"do required field validation for ${methodSymbol.typeSignature}") + val apiVersion = ApiVersionHolder.getApiVersion + validateRequiredFields(connectorMethodResult, methodSymbol.returnType, apiVersion) + } + val intercept: InvocationHandler = new InvocationHandler { override def invoke(proxy: AnyRef, method: Method, args: Array[AnyRef]): AnyRef = { if (method.getReturnType.getName == "scala.concurrent.Future" && !canOpenFuture(method.getName)) { throw new RuntimeException(ServiceIsTooBusy + s"Current Service(${method.getName})") + } else if (method.getName.contains("$default$") || ConnectorProxy.isInheritedMember(method)) { + delegateToStub(method, args) } else { - if (method.getName.contains("$default$") || ConnectorProxy.isInheritedMember(method)) { - // The empty Connector implements both: the $default$ accessors it inherits, and the - // members Connector itself does not declare. Routing the latter would look them up as - // connector calls - and NPE on the way, since args is null for a no-arg method. - val connectorMethodResult = method.invoke(StubConnector, args:_*) - if (connectorMethodResult.isInstanceOf[Future[_]] && canOpenFuture(method.getName)) { - FutureUtil.futureWithLimits(connectorMethodResult.asInstanceOf[Future[_]], method.getName) - } - connectorMethodResult - } else { - val methodName = method.getName - val argNameToValue: Array[(String, AnyRef)] = method.getParameters.map(_.getName).zip(args) - // TODO: getConnectorNameAndMethodRouting is also called inside invokeMethod. - // Consider refactoring invokeMethod to accept a pre-resolved connectorName to avoid the duplicate lookup. - val (_, connectorName) = getConnectorNameAndMethodRouting(methodName, argNameToValue) - - // Extract correlationId from CallContext before entering any Future callback, - // because Lift's S.containerSession is unavailable in async contexts. - val correlationId: String = args.collectFirst { - case Some(cc: CallContext) => cc.correlationId - case Full(cc: CallContext) => cc.correlationId - }.getOrElse(getCorrelationId()) // fallback to Lift session if no CallContext in args - - // Record outbound (before call) - ConnectorCountsRedis.incrementOutbound(connectorName, methodName) - val t0 = System.currentTimeMillis() - - val (connectorMethodResult, methodSymbol) = invokeMethod(method, args) - - // Track metrics for Future results - if (connectorMethodResult.isInstanceOf[Future[_]]) { - val future = connectorMethodResult.asInstanceOf[Future[Any]] - future.onComplete { result => - val duration = System.currentTimeMillis() - t0 - val isSuccess = result match { - case TrySuccess(value) => !isFailureBox(value) - case TryFailure(_) => false - } - - // Record inbound - ConnectorCountsRedis.incrementInbound(connectorName, methodName, isSuccess) - - // Record detailed metric to DB - if (getPropsAsBoolValue("write_connector_metrics", false)) { - val params = extractKeyParams(args) - Future { - ConnectorMetricsProvider.metrics.vend.saveConnectorMetric( - connectorName, methodName, correlationId, now, duration, params, isSuccess) - } - } - - // Record connector trace (outbound/inbound messages) - if (getPropsAsBoolValue("write_connector_trace", false)) { - val outbound = serializeOutboundArgs(method, args) - val inbound = serializeInboundResult(result) - val correlationId = getCorrelationId() - val (detailUserId, detailHttpVerb, detailApiUrl) = extractCallContextInfo(args) - val bankIdValue = extractBankIdFromArgs(args) - Future { - ConnectorTraceProvider.saveConnectorTrace( - correlationId, connectorName, methodName, bankIdValue, - outbound, inbound, now, duration, isSuccess, - detailUserId, detailHttpVerb, detailApiUrl) - } - } - } - } else { - // Non-future (legacy Box) result - track synchronously - val duration = System.currentTimeMillis() - t0 - val isSuccess = !isFailureBox(connectorMethodResult) - - ConnectorCountsRedis.incrementInbound(connectorName, methodName, isSuccess) - - if (getPropsAsBoolValue("write_connector_metrics", false)) { - val params = extractKeyParams(args) - Future { - ConnectorMetricsProvider.metrics.vend.saveConnectorMetric( - connectorName, methodName, correlationId, now, duration, params, isSuccess) - } - } - - // Record connector trace (outbound/inbound messages) - if (getPropsAsBoolValue("write_connector_trace", false)) { - val outbound = serializeOutboundArgs(method, args) - val inbound = serializeInboundResult(TrySuccess(connectorMethodResult)) - val correlationId = getCorrelationId() - val (detailUserId, detailHttpVerb, detailApiUrl) = extractCallContextInfo(args) - val bankIdValue = extractBankIdFromArgs(args) - Future { - ConnectorTraceProvider.saveConnectorTrace( - correlationId, connectorName, methodName, bankIdValue, - outbound, inbound, now, duration, isSuccess, - detailUserId, detailHttpVerb, detailApiUrl) - } - } - } - - if (connectorMethodResult.isInstanceOf[Future[_]] && canOpenFuture(method.getName)) { - FutureUtil.futureWithLimits(connectorMethodResult.asInstanceOf[Future[_]], method.getName) - } - logger.debug(s"do required field validation for ${methodSymbol.typeSignature}") - val apiVersion = ApiVersionHolder.getApiVersion - validateRequiredFields(connectorMethodResult, methodSymbol.returnType, apiVersion) - } + routeToConnector(method, args) } } } From e5ff924f3328ccad31c6f0ead9b1b963215e78d0 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 25 Aug 2026 22:08:28 +0200 Subject: [PATCH 06/59] fix: namespace the cache by serialization identity, so one version cannot read another's bytes An empty List, written to Redis by chill 0.9.3 on Scala 2.12, decodes under chill 0.9.5 on 2.13 into a scala.collection.immutable.Queue. The decode SUCCEEDS. It is the call site, whose signature says List, that dies: class scala.collection.immutable.Queue cannot be cast to class scala.collection.immutable.List Measured on GET /management/dynamic-message-docs and GET /management/connector-methods: 200 on 2.12, 500 on 2.13 reading the entry 2.12 wrote, and correct in either version running alone. The 500 lasts the whole TTL, because a read that throws does not evict the key. A rolling upgrade, or any upgrade against a warm Redis, produces exactly this. The migration note anticipated stale entries and described the consequence as a cold cache. For values that fail to decode that is right. This is the case that does not fail: it returns the wrong type, and no log line anywhere says so. Prefixing the key is the fix rather than casting at the call sites. There are eight List-returning memoized methods today and the same drift can hit any other type, but more to the point, no amount of care at a call site makes bytes already in Redis readable. With the prefix, another build's entries are not addressable at all and age out on their own TTL -- which is what "cold cache after rollout" was supposed to mean. The prefix carries the Scala binary version, the axis that moved here, plus obp.cache.serialization.version for the case it does not cover: a dependency upgrade that changes the encoding while the Scala version stays put, which is what chill 0.9.3 to 0.9.5 would have been on its own. CacheSerializationNamespaceTest pins the property, not the string. Asserting "obpser1-scala2.13" would say nothing about whether isolation holds and would turn every legitimate bump into a test edit. It also asserts the reverse control -- one namespace must still read its OWN entries -- because an isolation that isolated everything would pass while disabling the cache. Verified end to end against two instances sharing one Redis: the two reproductions now answer 200, the reverse direction still works, and a second call within one version reuses its key rather than writing a new one. --- .../src/main/scala/code/api/cache/Redis.scala | 46 +++++++- .../CacheSerializationNamespaceTest.scala | 101 ++++++++++++++++++ 2 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 obp-api/src/test/scala/code/api/cache/CacheSerializationNamespaceTest.scala diff --git a/obp-api/src/main/scala/code/api/cache/Redis.scala b/obp-api/src/main/scala/code/api/cache/Redis.scala index 05208e3360..cf1bdb89b4 100644 --- a/obp-api/src/main/scala/code/api/cache/Redis.scala +++ b/obp-api/src/main/scala/code/api/cache/Redis.scala @@ -7,7 +7,7 @@ import code.util.Helper.MdcLoggable import com.openbankproject.commons.ExecutionContext.Implicits.global import redis.clients.jedis.{Jedis, JedisPool, JedisPoolConfig} import scalacache.memoization.{cacheKeyExclude, memoizeF, memoizeSync} -import scalacache.{Cache, Flags} +import scalacache.{Cache, CacheConfig, DefaultCacheKeyBuilder, Flags} import scalacache.redis.RedisCache import scalacache.serialization.{Codec, FailedToDecode} import redis.clients.jedis.{Jedis, JedisPool, JedisPoolConfig} @@ -321,6 +321,50 @@ object Redis extends MdcLoggable { // building one per call only put two allocations in front of every cache read on the request // path. RedisCache is a thin wrapper over the pool built above and opens nothing of its own, so // the pool, its authentication and its SSL configuration stay shared. + /** + * The serialization identity these cached bytes were produced under. + * + * Cache entries are Kryo-encoded, and what Kryo produces depends on the Scala library and the + * chill build that encoded it. Two OBP-API versions compiled against different ones therefore + * write mutually unreadable bytes into the same keys -- and "unreadable" is the optimistic + * case. Measured across the 2.12 -> 2.13 migration: an EMPTY `List`, written by chill 0.9.3, + * decodes under 0.9.5 into a `scala.collection.immutable.Queue`. That decode SUCCEEDS. It is + * only at the call site, whose signature says `List`, that it fails -- + * + * class scala.collection.immutable.Queue cannot be cast to + * class scala.collection.immutable.List + * + * -- so the caller gets a 500 rather than a cache miss, and gets it for the whole TTL, because + * a failed read does not evict the entry. Reproduced on `GET /management/dynamic-message-docs` + * and `GET /management/connector-methods`: 200 on 2.12, 500 on 2.13 reading 2.12's entry, and + * fine in either version on its own. That is a rolling upgrade, or any upgrade against a warm + * Redis. + * + * The migration note anticipated the risk and described the consequence as a cold cache. For + * values that fail to decode that is exactly right. This handles the ones that do not fail. + * + * Namespacing the key is the fix rather than casting defensively at each call site: there are + * eight `List`-returning memoized methods today, the same drift can hit any other type, and no + * amount of care at the call sites can make bytes already in Redis readable. Entries written by + * another version simply stop being addressable and age out on their own TTL. + * + * The Scala binary version is the axis that moved here and is the one derived automatically. + * `obp.cache.serialization.version` is for the case it does not cover -- a dependency upgrade + * that changes the encoding without changing the Scala version, which is what chill 0.9.3 to + * 0.9.5 would have been on its own. Bump it in that situation; the cost is one cold cache. + */ + private val serializationNamespace: String = { + val scalaBinary = scala.util.Properties.versionNumberString.split('.').take(2).mkString(".") + val manual = APIUtil.getPropsValue("obp.cache.serialization.version", "1") + s"obpser$manual-scala$scalaBinary" + } + + // Prefixing happens here, in the key builder, rather than at the call sites: scalacache derives + // the rest of the key from the enclosing method and its arguments, and every caller goes through + // it. `memoizeSync` and `memoizeF` both read this same implicit config. + implicit val cacheConfig: CacheConfig = + CacheConfig(cacheKeyBuilder = DefaultCacheKeyBuilder(keyPrefix = Some(serializationNamespace))) + private val sharedCache: Cache[Any] = RedisCache[Any](jedisPool) private def cacheFor[A]: Cache[A] = sharedCache.asInstanceOf[Cache[A]] diff --git a/obp-api/src/test/scala/code/api/cache/CacheSerializationNamespaceTest.scala b/obp-api/src/test/scala/code/api/cache/CacheSerializationNamespaceTest.scala new file mode 100644 index 0000000000..cc39d211be --- /dev/null +++ b/obp-api/src/test/scala/code/api/cache/CacheSerializationNamespaceTest.scala @@ -0,0 +1,101 @@ +package code.api.cache + +import code.setup.RedisTestTarget +import org.scalatest.{FlatSpec, Matchers} +import scalacache.{CacheConfig, DefaultCacheKeyBuilder} + +/** + * Two OBP-API versions must not read each other's cached bytes. + * + * The defect this pins is not hypothetical and is not a decode failure. An empty `List`, written + * by chill 0.9.3 on Scala 2.12, decodes under chill 0.9.5 on 2.13 into a + * `scala.collection.immutable.Queue`. The decode SUCCEEDS; the call site, whose signature says + * `List`, is where it dies: + * + * class scala.collection.immutable.Queue cannot be cast to + * class scala.collection.immutable.List + * + * Measured on `GET /management/dynamic-message-docs` and `GET /management/connector-methods`: + * 200 on 2.12, 500 on 2.13 reading the entry 2.12 wrote, and correct in either version alone. + * The 500 lasts the whole TTL, because a read that throws does not evict the key. A rolling + * upgrade, or any upgrade against a warm Redis, produces exactly this. + * + * `Redis.serializationNamespace` prefixes every memoized key with the Scala binary version and a + * manually bumpable counter, so entries from another build are not addressable at all and expire + * on their own. + * + * ── What is asserted ── + * + * The PROPERTY, not the string. Asserting the current prefix would pin `obpser1-scala2.13`, which + * says nothing about whether isolation holds and turns every legitimate bump into a test edit. + * What has to stay true is that two different namespaces cannot see each other's entries, and + * that one namespace still sees its own -- an isolation that isolated everything, including a + * version from itself, would "pass" while disabling the cache entirely. + * + * These run against a real Redis. RedisTestTarget cancels them when none is reachable, and + * OBP_TEST_REDIS_REQUIRED=true turns that cancel into a failure so CI cannot lose the check + * silently. + */ +class CacheSerializationNamespaceTest extends FlatSpec with Matchers { + + private def keyFor(namespace: String, callerKey: String): String = + CacheConfig(cacheKeyBuilder = DefaultCacheKeyBuilder(keyPrefix = Some(namespace))) + .cacheKeyBuilder.toCacheKey(Seq(callerKey)) + + "the derived cache key" should "differ between two serialization namespaces" in { + val caller = "code.example.Provider.getAll(Some(bank))" + val a = keyFor("obpser1-scala2.12", caller) + val b = keyFor("obpser1-scala2.13", caller) + + withClue(s"2.12 key <$a> and 2.13 key <$b> are the same, so a 2.13 instance would read the " + + s"bytes a 2.12 instance wrote -- which is the defect this exists to prevent. ") { + a should not equal b + } + a should include("2.12") + b should include("2.13") + } + + it should "also differ when only the manual counter is bumped" in { + // The Scala version does not move for a dependency upgrade that changes the encoding -- + // chill 0.9.3 to 0.9.5 on its own would not have. The counter is the escape hatch for that, + // and it is only an escape hatch if it actually changes the key. + val caller = "code.example.Provider.getAll(Some(bank))" + keyFor("obpser1-scala2.13", caller) should not equal keyFor("obpser2-scala2.13", caller) + } + + it should "stay stable for one namespace, or nothing would ever be a cache hit" in { + val caller = "code.example.Provider.getAll(Some(bank))" + keyFor("obpser1-scala2.13", caller) shouldBe keyFor("obpser1-scala2.13", caller) + } + + "the namespace this build uses" should "name the Scala binary version it was compiled against" in { + // Derived, not asserted verbatim: the point is that it tracks the axis that actually moved. + val expected = scala.util.Properties.versionNumberString.split('.').take(2).mkString(".") + val probe = keyFor(s"obpser1-scala$expected", "x") + probe should include(expected) + } + + "a real Redis" should "not return an entry written under a different namespace" in { + RedisTestTarget.requireReachable(Redis.isRedisReady, "the cross-namespace isolation check") + + val caller = s"code.example.Probe.roundTrip(${System.nanoTime()})" + val oldKey = keyFor("obpser1-scalaOLD", caller) + val newKey = keyFor("obpser1-scalaNEW", caller) + + import code.api.JedisMethod + try { + Redis.use(JedisMethod.SET, oldKey, Some(60), Some("written-by-the-other-version")) + + withClue("the new namespace found the old namespace's entry -- the prefix is not isolating ") { + Redis.use(JedisMethod.GET, newKey, None, None) shouldBe None + } + withClue("the old namespace could not read back its OWN entry, so this test proved nothing " + + "about isolation -- it would pass with the cache switched off entirely ") { + Redis.use(JedisMethod.GET, oldKey, None, None) shouldBe Some("written-by-the-other-version") + } + } finally { + Redis.use(JedisMethod.DELETE, oldKey, None, None) + Redis.use(JedisMethod.DELETE, newKey, None, None) + } + } +} From d2849efefa4651a1de4b239a29fc862413193122 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 25 Aug 2026 22:37:24 +0200 Subject: [PATCH 07/59] feat: apply idempotency to every route tree, not only v7.0.0 IdempotencyMiddleware was mounted on one of nineteen route trees. The other eighteen carry 73 mutating payment endpoints between them -- UK Open Banking v3.1 and v4.0.1, Berlin Group v1.3 and v2, and the v4/v5.1/v6 core versions -- so a client retrying a payment on any of them repeated the payment. Tests first, because the middleware had none. IdempotencyMiddlewareTest pins the four properties a caller relies on when retrying: a same-key retry does not execute twice, the same key with a different body is refused rather than served the first response, one consumer's key cannot reach another's, and a 5xx is not cached so a retry actually retries. Wiring it up then failed end to end, and the way it failed is the reason this commit also changes the middleware. Http4sApp composes the version trees with `.orElse`, where OptionT.none means "not mine, try the next one". runRoutes called getOrElseF(404), turning a miss into an answer and terminating the chain: measured on POST /obp/v3.1.0/management/method_routings, which returned 201 without an Idempotency-Key and 404 with one. That was invisible while the middleware lived only on v7, the last link. It now returns the miss unchanged, and gives back the lock it took before running the routes -- otherwise a path this tree does not serve would hold the key for the lock's full 60s and refuse the request entitled to use it. Both new properties are pinned. Neither was reachable from the original unit tests: their inner routes are hand-built and have no fallthrough chain, so only running a real instance behind the real router could show it. Installation now has two documented requirements, both tested: inside ResourceDocMiddleware, because the body hash comes from CallContext.httpBody and without it every payload hashes alike -- which would return the first caller's receipt for a second, different payment -- and on every tree, because of the above. Verified against a live instance on three non-v7 payment paths: v3.1.0 method_routings, UK OB v4.0.1 domestic-payments and domestic-payment-consents, and Berlin Group v2 sepa-credit-transfers. Each: first call executes, replay returns the cached response with Idempotency-Replay: true and no second write, a changed body under the same key gives 409, and a call without a key is unaffected. --- .../UKOpenBanking/v2_0_0/Http4sUKOBv200.scala | 3 +- .../UKOpenBanking/v3_1_0/Http4sUKOBv310.scala | 3 +- .../UKOpenBanking/v4_0_1/Http4sUKOBv401.scala | 3 +- .../api/berlin/group/v1_3/Http4sBGv13.scala | 3 +- .../code/api/berlin/group/v2/Http4sBGv2.scala | 3 +- .../util/http4s/IdempotencyMiddleware.scala | 108 ++++++++++++------ .../scala/code/api/v1_2_1/Http4s121.scala | 3 +- .../scala/code/api/v1_3_0/Http4s130.scala | 3 +- .../scala/code/api/v1_4_0/Http4s140.scala | 3 +- .../scala/code/api/v2_0_0/Http4s200.scala | 3 +- .../scala/code/api/v2_1_0/Http4s210.scala | 3 +- .../scala/code/api/v2_2_0/Http4s220.scala | 3 +- .../scala/code/api/v3_0_0/Http4s300.scala | 3 +- .../scala/code/api/v3_1_0/Http4s310.scala | 3 +- .../scala/code/api/v4_0_0/Http4s400.scala | 3 +- .../scala/code/api/v5_0_0/Http4s500.scala | 3 +- .../scala/code/api/v5_1_0/Http4s510.scala | 4 +- .../scala/code/api/v6_0_0/Http4s600.scala | 4 +- .../http4s/IdempotencyMiddlewareTest.scala | Bin 0 -> 14085 bytes 19 files changed, 104 insertions(+), 57 deletions(-) create mode 100644 obp-api/src/test/scala/code/api/util/http4s/IdempotencyMiddlewareTest.scala diff --git a/obp-api/src/main/scala/code/api/UKOpenBanking/v2_0_0/Http4sUKOBv200.scala b/obp-api/src/main/scala/code/api/UKOpenBanking/v2_0_0/Http4sUKOBv200.scala index f7a58e2c74..b6a2c56045 100644 --- a/obp-api/src/main/scala/code/api/UKOpenBanking/v2_0_0/Http4sUKOBv200.scala +++ b/obp-api/src/main/scala/code/api/UKOpenBanking/v2_0_0/Http4sUKOBv200.scala @@ -4,6 +4,7 @@ import cats.data.{Kleisli, OptionT} import cats.effect._ import code.api.util.APIUtil.ResourceDoc import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.util.Helper.MdcLoggable import com.openbankproject.commons.util.ApiVersion import org.http4s._ @@ -33,5 +34,5 @@ object Http4sUKOBv200 extends MdcLoggable { Http4sUKOBv200AIS.routes(req) } - val wrappedRoutes: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allRoutes) + val wrappedRoutes: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes)) } diff --git a/obp-api/src/main/scala/code/api/UKOpenBanking/v3_1_0/Http4sUKOBv310.scala b/obp-api/src/main/scala/code/api/UKOpenBanking/v3_1_0/Http4sUKOBv310.scala index 56a3c403b1..49c41d3573 100644 --- a/obp-api/src/main/scala/code/api/UKOpenBanking/v3_1_0/Http4sUKOBv310.scala +++ b/obp-api/src/main/scala/code/api/UKOpenBanking/v3_1_0/Http4sUKOBv310.scala @@ -4,6 +4,7 @@ import cats.data.{Kleisli, OptionT} import cats.effect._ import code.api.util.APIUtil.ResourceDoc import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.util.Helper.MdcLoggable import com.openbankproject.commons.util.ApiVersion import org.http4s._ @@ -74,5 +75,5 @@ object Http4sUKOBv310 extends MdcLoggable { .orElse(Http4sUKOBv310InternationalStandingOrders.routes(req)) } - val wrappedRoutes: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allRoutes) + val wrappedRoutes: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes)) } diff --git a/obp-api/src/main/scala/code/api/UKOpenBanking/v4_0_1/Http4sUKOBv401.scala b/obp-api/src/main/scala/code/api/UKOpenBanking/v4_0_1/Http4sUKOBv401.scala index c40362d530..4f9985f712 100644 --- a/obp-api/src/main/scala/code/api/UKOpenBanking/v4_0_1/Http4sUKOBv401.scala +++ b/obp-api/src/main/scala/code/api/UKOpenBanking/v4_0_1/Http4sUKOBv401.scala @@ -5,6 +5,7 @@ import cats.effect._ import code.api.util.APIUtil import code.api.util.APIUtil.ResourceDoc import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.util.Helper.MdcLoggable import com.openbankproject.commons.util.ApiVersion import org.http4s._ @@ -62,5 +63,5 @@ object Http4sUKOBv401 extends MdcLoggable { routes(req).map(_.putHeaders(Header.Raw(fapiInteractionIdHeader, interactionId))) } - val wrappedRoutes: HttpRoutes[IO] = withFapiInteractionId(ResourceDocMiddleware.apply(resourceDocs)(allRoutes)) + val wrappedRoutes: HttpRoutes[IO] = withFapiInteractionId(ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes))) } diff --git a/obp-api/src/main/scala/code/api/berlin/group/v1_3/Http4sBGv13.scala b/obp-api/src/main/scala/code/api/berlin/group/v1_3/Http4sBGv13.scala index a2d83e2737..a50c74b563 100644 --- a/obp-api/src/main/scala/code/api/berlin/group/v1_3/Http4sBGv13.scala +++ b/obp-api/src/main/scala/code/api/berlin/group/v1_3/Http4sBGv13.scala @@ -5,6 +5,7 @@ import cats.effect._ import code.api.berlin.group.ConstantsBG import code.api.util.APIUtil.ResourceDoc import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.util.Helper.MdcLoggable import org.http4s._ @@ -35,5 +36,5 @@ object Http4sBGv13 extends MdcLoggable { .orElse(Http4sBGv13SigningBaskets.routes(req)) } - val wrappedRoutes: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allRoutes) + val wrappedRoutes: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes)) } diff --git a/obp-api/src/main/scala/code/api/berlin/group/v2/Http4sBGv2.scala b/obp-api/src/main/scala/code/api/berlin/group/v2/Http4sBGv2.scala index 9e2d1ce640..f62ec51171 100644 --- a/obp-api/src/main/scala/code/api/berlin/group/v2/Http4sBGv2.scala +++ b/obp-api/src/main/scala/code/api/berlin/group/v2/Http4sBGv2.scala @@ -5,6 +5,7 @@ import cats.effect._ import code.api.berlin.group.ConstantsBG import code.api.util.APIUtil.ResourceDoc import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.util.Helper.MdcLoggable import org.http4s._ @@ -27,5 +28,5 @@ object Http4sBGv2 extends MdcLoggable { .orElse(Http4sBGv2PIIS.routes(req)) } - val wrappedRoutes: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allRoutes) + val wrappedRoutes: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes)) } diff --git a/obp-api/src/main/scala/code/api/util/http4s/IdempotencyMiddleware.scala b/obp-api/src/main/scala/code/api/util/http4s/IdempotencyMiddleware.scala index f9e9c5032d..53e7bf6b37 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/IdempotencyMiddleware.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/IdempotencyMiddleware.scala @@ -44,9 +44,22 @@ import java.util.Base64 * Resilience: any Redis error is logged and the request is allowed to proceed * unchanged — the middleware never blocks traffic on cache outages. * - * The middleware should be installed INSIDE ResourceDocMiddleware so the - * CallContext (and therefore the consumer id) is populated before the scope - * key is computed. + * ── Where it may be installed ── + * + * INSIDE ResourceDocMiddleware, on every version's route tree. Both halves matter. + * + * Inside, because the scope key and the request-body hash both come from the CallContext, and + * ResourceDocMiddleware is what populates it. Mounted outside, `bodyFromCallContextOrEmpty` + * returns "" for every request, so two DIFFERENT payloads sent under one key hash identically -- + * the conflict check below silently stops working and the second caller receives the first + * caller's response. On a payment endpoint that is the difference between "your retry was + * deduplicated" and "your second, different payment returned someone else's receipt". + * + * On every tree, because Http4sApp composes the versions with `.orElse` and a tree signals "not + * mine" by returning OptionT.none. This middleware therefore has to pass a miss through + * unchanged; an earlier version answered 404 in that case, which terminated the chain -- measured + * on POST /obp/v3.1.0/management/method_routings, which answered 201 without an Idempotency-Key + * and 404 with one. IdempotencyMiddlewareTest pins both properties. */ object IdempotencyMiddleware extends MdcLoggable { @@ -81,6 +94,8 @@ object IdempotencyMiddleware extends MdcLoggable { } else { val key = keyOpt.get if (!isValidKey(key)) { + // A malformed header is a client error whatever the path resolves to, so this + // one deliberately does NOT fall through. OptionT.liftF(invalidKeyResponse(key)) } else { val scope = scopeFor(req) @@ -88,7 +103,17 @@ object IdempotencyMiddleware extends MdcLoggable { val responseKey = ResponseKeyPrefix + scope + ":" + key val lockKey = LockKeyPrefix + scope + ":" + key - OptionT.liftF(handle(req, routes, responseKey, lockKey, bodyHash)) + // OptionT, not OptionT.liftF: a route MISS has to stay a miss. + // + // Every version's routes are one link in a fallthrough chain -- Http4sApp composes them + // with `.orElse`, and `OptionT.none` is how a tree says "not mine, try the next one". + // Wrapping with liftF made this middleware answer 404 on behalf of a tree that simply + // did not serve the path, which terminated the chain: measured on + // `POST /obp/v3.1.0/management/method_routings`, the request answered 201 without an + // Idempotency-Key and 404 with one, because the first tree it passed through swallowed + // the miss. So the middleware could only ever be installed on the last link. Preserving + // the miss is what makes it safe to install on all of them. + OptionT(handle(req, routes, responseKey, lockKey, bodyHash)) } } } @@ -99,16 +124,16 @@ object IdempotencyMiddleware extends MdcLoggable { responseKey: String, lockKey: String, requestBodyHash: String - ): IO[Response[IO]] = { + ): IO[Option[Response[IO]]] = { IO.blocking(readResponseKey(responseKey)).attempt.flatMap { case Right(Some(envelope)) => if (envelope.requestBodyHash == requestBodyHash) { - IO.pure(rebuildResponse(envelope, replay = true)) + IO.pure(Some(rebuildResponse(envelope, replay = true))) } else { conflictResponse( "Idempotency-Key replayed with a different request body. " + "Use a fresh key for a different request." - ) + ).map(Some(_)) } case Right(None) => @@ -118,7 +143,7 @@ object IdempotencyMiddleware extends MdcLoggable { case Right(false) => conflictResponse( "Idempotent operation already in flight for this Idempotency-Key." - ) + ).map(Some(_)) case Left(t) => logger.warn(s"Idempotency lock unavailable (Redis): ${t.getMessage}") runRoutes(req, routes) @@ -136,41 +161,48 @@ object IdempotencyMiddleware extends MdcLoggable { responseKey: String, lockKey: String, requestBodyHash: String - ): IO[Response[IO]] = { - runRoutes(req, routes).flatMap { resp => - // Drain body so we can both cache and re-emit it. - resp.body.compile.toVector.flatMap { vec => - val bodyBytes = vec.toArray - val rebuilt = resp.withBodyStream(fs2.Stream.emits(bodyBytes).covary[IO]) - - val storeOrReleaseLock: IO[Unit] = - if (resp.status.code >= 500) { - // Don't cache transient failures; release the lock so client can retry. - IO.blocking(deleteKey(lockKey)).attempt.map(_ => ()) - } else { - val envelope = Envelope( - status = resp.status.code, - contentType = resp.headers.get(CIString("Content-Type")).map(_.head.value), - bodyB64 = Base64.getEncoder.encodeToString(bodyBytes), - requestBodyHash = requestBodyHash - ) - IO.blocking { - writeResponseKey(responseKey, envelope) - deleteKey(lockKey) - }.attempt.map { e => - e.left.foreach(t => - logger.warn(s"Failed to cache idempotent response: ${t.getMessage}") + ): IO[Option[Response[IO]]] = { + runRoutes(req, routes).flatMap { + // The lock was taken before the routes ran, so a miss has to give it back -- otherwise a + // path this tree does not serve would hold the key locked for its full 60s TTL and a + // genuine request carrying that key would be refused with 409. + case None => IO.blocking(deleteKey(lockKey)).attempt.as(None) + case Some(resp) => + // Drain body so we can both cache and re-emit it. + resp.body.compile.toVector.flatMap { vec => + val bodyBytes = vec.toArray + val rebuilt = resp.withBodyStream(fs2.Stream.emits(bodyBytes).covary[IO]) + + val storeOrReleaseLock: IO[Unit] = + if (resp.status.code >= 500) { + // Don't cache transient failures; release the lock so client can retry. + IO.blocking(deleteKey(lockKey)).attempt.map(_ => ()) + } else { + val envelope = Envelope( + status = resp.status.code, + contentType = resp.headers.get(CIString("Content-Type")).map(_.head.value), + bodyB64 = Base64.getEncoder.encodeToString(bodyBytes), + requestBodyHash = requestBodyHash ) - () + IO.blocking { + writeResponseKey(responseKey, envelope) + deleteKey(lockKey) + }.attempt.map { e => + e.left.foreach(t => + logger.warn(s"Failed to cache idempotent response: ${t.getMessage}") + ) + () + } } - } - storeOrReleaseLock.as(rebuilt) - } + storeOrReleaseLock.as(Some(rebuilt)) + } } } - private def runRoutes(req: Request[IO], routes: HttpRoutes[IO]): IO[Response[IO]] = - routes.run(req).getOrElseF(IO.pure(Response[IO](Status.NotFound))) + // `.value`, not `getOrElseF(404)` -- see the comment on the OptionT in `apply`. Converting a + // miss into a 404 here is what terminated the version fallthrough chain. + private def runRoutes(req: Request[IO], routes: HttpRoutes[IO]): IO[Option[Response[IO]]] = + routes.run(req).value // ── Validation ───────────────────────────────────────────────────────── diff --git a/obp-api/src/main/scala/code/api/v1_2_1/Http4s121.scala b/obp-api/src/main/scala/code/api/v1_2_1/Http4s121.scala index 3d7ceb4127..891231fd92 100644 --- a/obp-api/src/main/scala/code/api/v1_2_1/Http4s121.scala +++ b/obp-api/src/main/scala/code/api/v1_2_1/Http4s121.scala @@ -10,6 +10,7 @@ import code.api.util.ApiTag._ import code.api.util.ErrorMessages._ import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps, callContextKey} import code.api.util.http4s.Http4sCallContextBuilder +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.http4s.ResourceDocMiddleware import code.api.util.newstyle.ViewNewStyle import code.api.util.{CallContext, CustomJsonFormats, NewStyle} @@ -2669,7 +2670,7 @@ object Http4s121 { } val allRoutesWithMiddleware: HttpRoutes[IO] = { - val middlewareWrapped = ResourceDocMiddleware.apply(resourceDocs)(allRoutes) + val middlewareWrapped = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes)) // bankById runs before middleware so it can return 400 (not 404) for unknown bank Kleisli[HttpF, Request[IO], Response[IO]] { req => bankById.run(req).orElse(middlewareWrapped.run(req)) diff --git a/obp-api/src/main/scala/code/api/v1_3_0/Http4s130.scala b/obp-api/src/main/scala/code/api/v1_3_0/Http4s130.scala index c3fe407a9e..27d0ac3d82 100644 --- a/obp-api/src/main/scala/code/api/v1_3_0/Http4s130.scala +++ b/obp-api/src/main/scala/code/api/v1_3_0/Http4s130.scala @@ -9,6 +9,7 @@ import code.api.util.ApiTag._ import code.api.util.ErrorMessages._ import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.NewStyle import code.api.v1_2_1.JSONFactory import com.github.dwickern.macros.NameOf.nameOf @@ -126,7 +127,7 @@ object Http4s130 { .orElse(getCardsForBank.run(req)) } - val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allOwnRoutes) + val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allOwnRoutes)) // ─── path-rewriting bridge: /obp/v1.3.0/… → /obp/v1.2.1/… ───────────── // Delegates to Http4s121 so all inherited v1.2.1 endpoints are served diff --git a/obp-api/src/main/scala/code/api/v1_4_0/Http4s140.scala b/obp-api/src/main/scala/code/api/v1_4_0/Http4s140.scala index 1f5c17a58a..d9c7a2404d 100644 --- a/obp-api/src/main/scala/code/api/v1_4_0/Http4s140.scala +++ b/obp-api/src/main/scala/code/api/v1_4_0/Http4s140.scala @@ -10,6 +10,7 @@ import code.api.util.ApiTag._ import code.api.util.ErrorMessages._ import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.{APIUtil, NewStyle} import code.api.v1_2_1.{JSONFactory, SuccessMessage} import code.atms.Atms @@ -475,7 +476,7 @@ object Http4s140 { .orElse(addCustomer.run(req)) } - val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allOwnRoutes) + val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allOwnRoutes)) // ─── path-rewriting bridge: /obp/v1.4.0/… → /obp/v1.3.0/… ────────────── // Delegates to Http4s130 so all inherited v1.3.0 and v1.2.1 endpoints are diff --git a/obp-api/src/main/scala/code/api/v2_0_0/Http4s200.scala b/obp-api/src/main/scala/code/api/v2_0_0/Http4s200.scala index 38b4915de1..bd65de6109 100644 --- a/obp-api/src/main/scala/code/api/v2_0_0/Http4s200.scala +++ b/obp-api/src/main/scala/code/api/v2_0_0/Http4s200.scala @@ -13,6 +13,7 @@ import code.api.util.ApiTag._ import code.api.util.ErrorMessages._ import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.newstyle.ViewNewStyle import code.api.util.{APIUtil, ApiRole, CustomJsonFormats, NewStyle} import code.api.v1_2_1.{JSONFactory => JSONFactory121, SuccessMessage} @@ -1586,7 +1587,7 @@ object Http4s200 { .orElse(elasticSearchMetrics.run(req)) } - val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allOwnRoutes) + val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allOwnRoutes)) // ─── path-rewriting bridge: /obp/v2.0.0/… → /obp/v1.4.0/… ────────────── // Delegates to Http4s140 so all inherited v1.4.0/v1.3.0/v1.2.1 endpoints are diff --git a/obp-api/src/main/scala/code/api/v2_1_0/Http4s210.scala b/obp-api/src/main/scala/code/api/v2_1_0/Http4s210.scala index 4081ac34b4..cd31ae8891 100644 --- a/obp-api/src/main/scala/code/api/v2_1_0/Http4s210.scala +++ b/obp-api/src/main/scala/code/api/v2_1_0/Http4s210.scala @@ -12,6 +12,7 @@ import code.api.util.ApiTag._ import code.api.util.ErrorMessages._ import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.newstyle.ViewNewStyle import code.api.util.{APIUtil, ApiRole, CallContext, CustomJsonFormats, NewStyle} import code.api.v1_2_1.{JSONFactory => JSONFactory121, SuccessMessage} @@ -1393,7 +1394,7 @@ object Http4s210 { .orElse(getMetrics.run(req)) } - val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allOwnRoutes) + val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allOwnRoutes)) // ─── path-rewriting bridge: /obp/v2.1.0/… → /obp/v2.0.0/… ────────────── // Delegates to Http4s200 so all inherited v2.0.0/v1.4.0/v1.3.0/v1.2.1 endpoints diff --git a/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala b/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala index f62b53c99a..fd0cb4dab4 100644 --- a/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala +++ b/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala @@ -13,6 +13,7 @@ import code.api.util.Glossary import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import java.util.Date import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.newstyle.ViewNewStyle import code.api.util.{APIUtil, CallContext, CustomJsonFormats, NewStyle} import code.api.v1_2_1.{CreateViewJsonV121, JSONFactory => JSONFactory121, UpdateViewJsonV121} @@ -1064,7 +1065,7 @@ object Http4s220 { .orElse(createCounterparty.run(req)) } - val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allOwnRoutes) + val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allOwnRoutes)) // ─── path-rewriting bridge: /obp/v2.2.0/… → /obp/v2.1.0/… ────────────── diff --git a/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala b/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala index 4641e6ab44..b5d6745188 100644 --- a/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala +++ b/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala @@ -15,6 +15,7 @@ import code.api.util.ApiTag._ import code.api.util.ErrorMessages._ import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.newstyle.ViewNewStyle import code.api.util.{APIUtil, CallContext, CustomJsonFormats, NewStyle} import code.api.v1_2_1.JSONFactory @@ -2303,7 +2304,7 @@ object Http4s300 { .orElse(bankById.run(req)) } - val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allOwnRoutes) + val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allOwnRoutes)) // ─── path-rewriting bridge: /obp/v3.0.0/… → /obp/v2.2.0/… ────────────── diff --git a/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala b/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala index 168094cbc3..fb97c8f4e3 100644 --- a/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala +++ b/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala @@ -17,6 +17,7 @@ import code.api.util.CertificateUtil import code.api.util.{ApiTrigger, Consent, Glossary, SecureRandomUtil} import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.newstyle.{BalanceNewStyle, ViewNewStyle} import code.api.util.{APIUtil, CallContext, CustomJsonFormats, NewStyle, OBPBankId, RateLimitingUtil} import code.api.v1_2_1.{JSONFactory, RateLimiting} @@ -5118,7 +5119,7 @@ object Http4s310 { .orElse(getObpConnectorLoopback.run(req)) } - val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allOwnRoutes) + val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allOwnRoutes)) // ─── path-rewriting bridge: /obp/v3.1.0/… → /obp/v3.0.0/… ────────────── diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala index d821649e0d..b427ae87a8 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala @@ -35,6 +35,7 @@ import code.api.v1_4_0.JSONFactory1_4_0 import code.DynamicEndpoint.DynamicEndpointSwagger import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.{APIUtil, CallContext, CustomJsonFormats, NewStyle} import code.api.v4_0_0.JSONFactory400._ import code.DynamicData.DynamicData @@ -11097,7 +11098,7 @@ object Http4s400 { .orElse(createUserInvitation.run(req)) } - lazy val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allOwnRoutes) + lazy val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allOwnRoutes)) // ─── nameOf-compatibility aliases ──────────────────────────────────────── // These vals have no Lift counterpart in Http4s400 but are referenced by diff --git a/obp-api/src/main/scala/code/api/v5_0_0/Http4s500.scala b/obp-api/src/main/scala/code/api/v5_0_0/Http4s500.scala index 1de94f93d0..dd1d323a2e 100644 --- a/obp-api/src/main/scala/code/api/v5_0_0/Http4s500.scala +++ b/obp-api/src/main/scala/code/api/v5_0_0/Http4s500.scala @@ -14,6 +14,7 @@ import code.api.util.ErrorMessages import code.api.util.ErrorMessages._ import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.newstyle.ViewNewStyle import code.api.util.{APIUtil, ConsentJWT, ConsentView, Consent, CustomJsonFormats, JwtUtil, NewStyle, OBPBankId, SecureRandomUtil} import code.api.v2_1_0.JSONFactory210 @@ -2353,7 +2354,7 @@ object Http4s500 { } val allRoutesWithMiddleware: HttpRoutes[IO] = - ResourceDocMiddleware.apply(resourceDocs)(allRoutes) + ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes)) // ─── path-rewriting bridge: /obp/v5.0.0/… → /obp/v4.0.0/… ───────────── // Cascades inherited (v1.2.1–v4.0.0) endpoints through the http4s versions diff --git a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala index 96e1d695d0..80d5e2b17a 100644 --- a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala +++ b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala @@ -13,7 +13,7 @@ import code.api.util.ApiTag._ import code.api.util.ErrorMessages import code.api.util.ErrorMessages._ import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} -import code.api.util.http4s.{ResourceDocMiddleware, ResourceDocMatcher} +import code.api.util.http4s.{IdempotencyMiddleware, ResourceDocMatcher, ResourceDocMiddleware} import code.api.util.newstyle.{BalanceNewStyle, RegulatedEntityAttributeNewStyle, ViewNewStyle} import code.api.util.newstyle.RegulatedEntityNewStyle.{createRegulatedEntityNewStyle, deleteRegulatedEntityNewStyle, getRegulatedEntitiesNewStyle, getRegulatedEntityByEntityIdNewStyle} import code.api.util.newstyle.Consumer.createConsumerNewStyle @@ -5326,7 +5326,7 @@ object Http4s510 { } val allRoutesWithMiddleware: HttpRoutes[IO] = - ResourceDocMiddleware.apply(resourceDocs)(allRoutes) + ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes)) // ─── path-rewriting bridge: /obp/v5.1.0/… → /obp/v5.0.0/… ───────────── lazy val v510ToV500Bridge: HttpRoutes[IO] = Kleisli[HttpF, Request[IO], Response[IO]] { req => diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index 01b6bae3a7..9674d249b1 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -32,7 +32,7 @@ import code.api.util.{APIUtil, CallContext, CustomJsonFormats, NewStyle} import code.api.util.ApiRole._ import code.api.util.ApiTag._ import code.api.util.ErrorMessages._ -import code.api.util.http4s.{ErrorResponseConverter, RequestScopeConnection, ResourceDocMiddleware, ResourceDocMatcher} +import code.api.util.http4s.{ErrorResponseConverter, IdempotencyMiddleware, RequestScopeConnection, ResourceDocMatcher, ResourceDocMiddleware} import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.newstyle.ViewNewStyle import code.api.v2_0_0.JSONFactory200 @@ -6308,7 +6308,7 @@ object Http4s600 { // Deferring index construction to first request (post object-init) lets every // registration land before the snapshot is taken. lazy val allRoutesWithMiddleware: HttpRoutes[IO] = - ResourceDocMiddleware.apply(resourceDocs)(allRoutes) + ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes)) // ─── path-rewriting bridge: /obp/v6.0.0/… → /obp/v5.1.0/… ───────────── // Targets v5.1.0; Http4s510 has its own working cascade down to v5.0.0 → v4.0.0 → … diff --git a/obp-api/src/test/scala/code/api/util/http4s/IdempotencyMiddlewareTest.scala b/obp-api/src/test/scala/code/api/util/http4s/IdempotencyMiddlewareTest.scala new file mode 100644 index 0000000000000000000000000000000000000000..80a135ae1138156bd94e249a0d642adc3161deb3 GIT binary patch literal 14085 zcmdU0e{b7Hn!dmNDF#Fim3tB`Cr-9?kpi~kwBBywuC3f2usEa?IkLExNbL-1MQ0ee z4{@LHKFL1MJ413L%dV5|9&U@>#36?>^Zt6?pI#*Cb#kRuT4p*<7I|D(`7EAR)#AYJ z?Bw%BX)2W_m5udeqSGoqJKy%SE^IQs%7+FlAFXyAT5iuHbxg!oK)pJPvax`>kQ;yX|uDl_x0C1>T5NcQg%gU zlC)CgM4e?CGOKivuHNNYHq*<*=)PK-yuw43mCHhn<=QH*RBlyX*b02rnM#V)68F_g z&#aEId9(k#-H0E&*#Ab&@N+p7Se?o4^RmW;GgTH!f5H`K zEBvUrvYKk{Utp(7<7TF;E3GP{vGHJ_ZhjXJ;~{@qPV*UFfX)(s7^U;8seF+p#;p3F zrdo0Xyqx#d?_Q{RT_qK{itQK4Y7QPMU1WQ6ZZ zi?xaju;f8nLNWz!19_DyX)m`(swp{38keTTXQjy?W85E%Di_*-9_-pz7Mz}9rT&y# z&@#yl)DY{q^iR`7Se{s`4bKxacBS-yp*+-{3BB1x{|XeWo^{L@W3jk zd|~*|47td%tY?`rS_FviDKTA{l@Qrba_Fz3)P9VwWr14eppM7eWUkeYFuV3^3Lc^;`Aa(O5ek`d$?HWg(u8kRXRS)c%Qxk(iyVAZO*&lEw=zlRV!r$ z{+wPfOI|yJ20n-3na1^xjI=9ng42%F+kX78?3QxWM z;`~(SH`?+5zI}UkaXIoYA@xQrV)eE(l*k5X;2S3ue%o9y)h+TwD;kbyJ5N|bteMVg zN(qi4i{w^rk-JyF-9^Y0gt9=ng7}DiYEwYYww!B%CE@rhxx3J@sG?XM0cQnUX@7!j zfIWzBQ7&qFCm}NwlgE5Pj{wgt;TB}^-d;*@7SlWx9a>IT4G;tG0XTFSg{4_sXbYlK z{iaMiKhx;=2vUYh)zG1~xGg3P{ie|4@Jk|0qTG&0^6oDv>LM%KM!k z|NL=udO7-dar)-$5`X^j!x?^jU6~s6f@h=zf^xfo3p6NTKB0D!fx%Tm@fug|Frabg?qBy^zq;MoQ3`HZozUqKq={|>ZsK;7Sn{fN-+ zf_82He0Khy__lixmlJt*i!T-Sh75l`hoKC(N7CPG^h^?VpPsAp|EJ3EB?PYr6V>pJKyJHlFe zBK)}|5QoYo5`B5bAgbBjJppBh2=Zuwe|D!*-QR;Y_dcol#Ul&o!X90~*dPGeLv>u1 zGYy-4jX<^rs45T)q?Dk;o0~JPn}iPIuUL5Dd?VlX99@0&!5|NljOr~MxVw9g1nxTH z0c9~W4@&U!62ONO-jCZD(O z5xy-={$GN++%;)-%_pk2!KhteB_T#R5KF;^fQ>z7pNvuPm?CgAtAI#ucT^#EK;)`> z1h50^`Pe?TG2!C`p*G+gl=fAcx}EOiY;T7#l?sg8jgd2+n^XAOx4pA-@X_dAD7xMy zvWnWq@5|~f@I3>0UPc#nak(ne9(P|ecd=6ta6ajobv+2|KrRC##k!ao0>Xf}D*qj+ zc4;;bi08?o_Ytp8ris~&=hsd(Hti$_GZwRCwdK@SLH)5)rz<4>0VQ!;MWEIt9XOvK z#c)4xyRTm(wScMv)v;saZp2vW>N{+b^Thqr#kv$d2H)%4=nE<`Jv|tXk7C{*f8#Mj zNJ(7)drex4n7GMnT}W)*;d^`PfB)yd@jt|G)()yET*-jd?E@2za#s6&e{jL@8X&k0(gwSF0+7tYQuMz_vTN+ zVBRS3_#($Q0KtUSKe+s;wS36i=)b^;K&?cm2g9L{83Q~3=kd*&`J2ZKHxOPVmw;Pv zB7DdWkCV(hW?9#nA{E_%l7V$(h>;Kt(onpbQBj=TL79={h=&R+ZU{^3tc@3A6%@&) z*c(i_7{D9h2@3-)haP9pm$E_>?O$+*0qGkQ_3U%`vYo^Y0%xcUHd257@?}TWugLvN z$-0h^bU%Oj>N92hI4zx*rUtmmg{uZ5`$zB$?W6?%-x_S|g;D!-mPI<+(kAOq@(+5- zPTS~N!eVDrVUkj3ARl-E22ElsbffF=Y9y-j2+P;lh9xqXtR%jzGiK^rmB@Ml*r}Eb zm-lwzwkw4`;^YG>zh)qh$i9K>=vXHPI+*S=#1EqZe-*DQt}y)s2i}IN@x%EvER^zY&V=#td`QQAWF_y&9YdWf`#!+zzS>lJ~m8q)p(rP+X7#YdXtuX8^kt7b-nt*cn7#{Ft2G z!W9;@lcv0D>oi4$XHttPu+p@~hgP}nUEgQtbRdlrSuYdL4!l zSFC>FxvW+N+oPPqF3&FzDEm0Uo3;&wh!0&V26#dx_8(&)ft{}iU5g`-&l;0+MFcdlP+wJT7sgiQ0u71MNO#yHy0;Wc0ZtIv zj_YNKD{8iPFW0(CBck^LeGr_B{={0t51j#YCu(M6n++Wgk0_C7^o5%JZ}L(aJgCMt z@}k9nCe&3PO3aEfP48!BUj{955&9_6G@r|@8T#~2aO8Ejxa*}dKch$j{Y6I zc6%-7KmDH%-mw9sNA4&)3I$R;;7@F9hV)R6)^!gDvH>ccPIPTE`gf`X3Rz9_zV(pz z@FZfd47S4RM5CT!U8Ar8S*GY#%Txu{SeAp(J#)=Z1S4$i$QT8b4QhA)j3A7_d!9&j zmJyA2AJFZ2u!ZP?y3fEbm_71OFPP7B#%uOWGTERH@*D=#O^06E=3NZ_WS=Y5l3$Oe zUs9Kah@epM`KIG(7#uwwIWw$xb1uKpCYtk(A3kqG!5cQKV`*Xgpr(C#1WeoXmcT`L zX-ohBy(p3bIBOS43U#`#c$me%Ui60N_}^QeIJ z^m;D+V#47(F*>{wfU$rXV5uyG(yV0%k7$BIY~lJIJs&l8!C9A>cDVybM&KL-a5=%| zCY-1(Vnm$+f!`Qjk*OY+IgBxWf5JXr_ zOAh?m#SYvQbhQ%{*0pj$fg4&$3cExEFWzjdr^yWSbC^nO93nVpIo638G)!Z(0+L-t zge3Dim&17y@{YW~R2%-o5d?j;SB@8ZoED6Sd-J5gbOD|5o@?EHB=?|boGJ2CJkN*w zU7&WaS0j7{ZOswFc6O*CzcfJ_Bv*5AAdE)X$MzN|Y#Lm431*c`dBWy_4a)PZ=K4mb zoS>BSJAFx3dd_Qop4#Nm9@sK);6E6z&_~PBOoLK;cp@*pq|4ln14O(cV8{i5PulKY zb2+R-1^bYDZ3Fnpec==E@5J?q!`8EFatUI#cXeIQhLx^Gf6=yo<#~%;L2ApH0~h|m zP0|#CC{3~mC@AKad}eDf-bFlc1Dtcr6L17@OEtuz!RMNRk93*e*y~!7i432v(R_#tc5-O`;k()<+efj44D#I7C(qM0hC2BYC{5x6^RG z;kViQmsoq#+j|05JjuA@|0C1hoVeJ~bCDn8H*Z6~F-E^vMGpeB07s~WZHFP{6tyT7 zOJE#u_I2Dlr$0Utfu}pWY_93c7h(W2J;6BmX1STeSXNXWCWHe8uyJ+XZS@1 zgYbt_$tgAg?xHU%Um%@@yOt^#;lqvS^NpB1rZD*P5^OJ*51F^11vwHFPuT8qyz~+;KLq5fC<`qP1v<&e*kL^kM;e){62oK0o`REQMmrmoc$*G zMHR%x2pBLe{?3hAzR9mxgMvppV^slFiC%R1`>_s0& zZwKEXI*?H(Op$!ACegF;zYd=xJ!%J@VuF2d|CM8f(>YotrScQ1#xh)piJ?P2ID96F z{^7H0m&G$QZ8&xz^G4uGFZjiz7%I_k_UgM?g;zuLyIe#7qisSCp83><`O{lz0SG8p zL1Xtf#e`rbuoYaEKM+6lso+ksjX%0EKx>=e01rg}RMs4u;^EE8rIl zpunA#F}Rg6e?MEo4Q)N1!|_qye73%ZAei3sH@OcVVVT=?WEd1~#gL)c+8wkLaDkBK XaXGr-m_U>=`zaAir#W{!cRPOr&31$4 literal 0 HcmV?d00001 From bb77c324dfd5d67b0aca8ced235db59999d1d6f9 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 25 Aug 2026 23:01:20 +0200 Subject: [PATCH 08/59] fix: make six endpoints declare the authentication they actually enforce The sweep found ten endpoints whose ResourceDoc disagreed with their behaviour. They are not one defect; they are four, and only six of the ten needed a product change. The other four were the sweep being wrong. Six docs corrected: getAllProductsV600, getAllApiProductsV600 -- the description still interpolated userAuthenticationMessage(!getProductsIsPublic), so with the prop true it published "authentication optional" while the route called withUser unconditionally. The route is deliberate: its own comment reads "(all banks; auth-required; cached)", and the api-products bucket records why -- "the v6 Lift conditional public-access path (getApiProductsIsPublic) is simplified -- public gating would be a Phase 3 follow-up if needed". So the doc is what was out of step. Restoring the conditional route would have been implementing that follow-up, which is a product decision and makes two endpoints public by default. getConfigProps, getConnectorTraces -- hand-written "Authentication is Required." The constructor matches userAuthenticationMessage(true) verbatim and this is not it, so nothing reached errorResponseBodies and both published as public. getMyApiCollectionEndpoint, getApiCollectionEndpoints -- these contradicted themselves: errorResponseBodies listed $AuthenticatedUserIsRequired while the description said userAuthenticationMessage(false). The description wins -- the constructor's second branch REMOVES the error body when the text says optional -- so an explicit declaration was being deleted by a text match. Four needed no product change, and AuthSweepTest is corrected instead: createConsentRequest, getConsentRequest and createVRPConsentRequest require an APPLICATION, not a user. All three answer OBP-20200 "The application cannot be identified", and createVRPConsentRequest says so in its own prose: "Client, Consumer or Application Authentication is mandatory for this endpoint". userAuthenticationMessage(false) is accurate. EndpointCatalog.needsAuthentication reproduces the middleware's predicate, which reads only errorResponseBodies and roles -- both about the user -- so it classified them public and then failed them for behaving as documented. A 401 now only fails the check when it is the user one; an application 401 is reported as an observation, named rather than swallowed, since resource-docs cannot express "needs an application" without authMode. verifyRequestSignResponse refuses with OBP-20311 "The Request is not signed" -- a third mechanism, which authMode cannot model either. Left failing on purpose. AuthSweepTest: 9 failures before, 2 after. The two that remain are verifyRequestSignResponse and createTransactionRequestFreeForm answering 500 where 403 was due, both already catalogued. --- .../scala/code/api/v4_0_0/Http4s400.scala | 4 +- .../scala/code/api/v6_0_0/Http4s600.scala | 8 ++-- .../scala/code/api/sweep/AuthSweepTest.scala | 39 ++++++++++++++++--- 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala index b427ae87a8..d6c1e7c8ef 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala @@ -7096,7 +7096,7 @@ object Http4s400 { "Get My Api Collection Endpoint", s"""Get Api Collection Endpoint By API_COLLECTION_NAME and OPERATION_ID. | - |${userAuthenticationMessage(false)} + |${userAuthenticationMessage(true)} |""".stripMargin, EmptyBody, apiCollectionEndpointJson400, @@ -7114,7 +7114,7 @@ object Http4s400 { "Get Api Collection Endpoints", s"""Get Api Collection Endpoints By API_COLLECTION_ID. | - |${userAuthenticationMessage(false)} + |${userAuthenticationMessage(true)} |""".stripMargin, EmptyBody, apiCollectionEndpointsJson400, diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index 9674d249b1..4927967ba2 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -8773,7 +8773,7 @@ object Http4s600 { | |9 user_id (if null ignore) | - |Authentication is Required. + |${userAuthenticationMessage(true)} | |""".stripMargin, EmptyBody, @@ -9868,7 +9868,7 @@ object Http4s600 { | |Optional query parameter `tag` — filter to products that have the given tag (e.g. `?tag=featured`). Tag matching is case-insensitive. | - |${userAuthenticationMessage(!getApiProductsIsPublic)}""".stripMargin, + |${userAuthenticationMessage(true)}""".stripMargin, EmptyBody, apiProductsJsonV600, List(UnknownError), @@ -9886,7 +9886,7 @@ object Http4s600 { | |Optional query parameter `tag` — filter to products that carry the given tag (e.g. `?tag=featured`). Tag matching is case-insensitive. Repeat `tag=` to require multiple tags. | - |${userAuthenticationMessage(!getProductsIsPublic)}""".stripMargin, + |${userAuthenticationMessage(true)}""".stripMargin, EmptyBody, productsJsonV600, List(UnknownError), @@ -13341,7 +13341,7 @@ object Http4s600 { |Properties with sensitive keys or values (containing ${APIUtil.sensitiveKeywords.mkString(", ")}) |are excluded from the response entirely. | - |Authentication is Required. + |${userAuthenticationMessage(true)} | |""".stripMargin, EmptyBody, diff --git a/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala b/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala index 285cac71a9..d919ade548 100644 --- a/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala +++ b/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala @@ -3,7 +3,7 @@ package code.api.sweep import cats.effect.IO import cats.effect.unsafe.IORuntime import code.api.util.APIUtil.ResourceDoc -import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, UserHasMissingRoles} +import code.api.util.ErrorMessages.{ApplicationNotIdentified, AuthenticatedUserIsRequired, UserHasMissingRoles} import code.api.util.http4s.Http4sApp import code.setup.{DefaultUsers, ServerSetupWithTestData} import fs2.Stream @@ -113,11 +113,40 @@ class AuthSweepTest extends ServerSetupWithTestData with DefaultUsers { else None } + /** + * A doc that asks for no USER may still ask for an APPLICATION, and that is not a defect. + * + * OBP has three ways to refuse an anonymous caller, and this check originally modelled one: + * + * OBP-20001 User not logged in -- user authentication + * OBP-20200 The application cannot be identified -- consumer/application authentication + * OBP-20311 The Request is not signed -- JWS request signing + * + * `EndpointCatalog.needsAuthentication` reproduces the middleware's predicate, which reads + * only errorResponseBodies and roles -- both about the user. So an endpoint that requires a + * consumer is classified "public" here and then fails this assertion for doing exactly what + * its doc says. Measured on createConsentRequest, getConsentRequest and + * createVRPConsentRequest: all three answer OBP-20200, and the last one spells it out in its + * own description -- "Client, Consumer or Application Authentication is mandatory for this + * endpoint". Their docs were right; this check was wrong. + * + * So a 401 is only a violation when it is the USER one. An application-auth 401 is reported + * as an observation instead of a failure -- named, not silently swallowed, because the doc + * still has no machine-readable way to say "needs an application" unless someone sets + * authMode, and a reader of resource-docs cannot tell. + */ private def checkPublicIsNot401(doc: ResourceDoc): Option[String] = { - val (code, _) = call(doc.requestVerb, EndpointCatalog.concretePath(doc), Map.empty) - if (code == 401) - Some(s"${describe(doc)} -- declares no authentication requirement yet answered 401 anonymously") - else None + val (code, json) = call(doc.requestVerb, EndpointCatalog.concretePath(doc), Map.empty) + val msg = messageOf(json) + if (code != 401) None + else if (msg.startsWith(ApplicationNotIdentified.take(9))) { + info(s"${describe(doc)} -- declares no user authentication and requires an APPLICATION " + + s"instead ($msg). The doc is accurate about the user; consider authMode = " + + s"ApplicationOnly so resource-docs can say so too.") + None + } else + Some(s"${describe(doc)} -- declares no authentication requirement yet answered 401 " + + s"anonymously with '$msg'") } private def checkNoRoleIs403(doc: ResourceDoc): Option[String] = { From dbb94da69e448500f8d31cd6b3cda70aa9efc8d9 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 25 Aug 2026 23:13:22 +0200 Subject: [PATCH 09/59] test: compare the class Kryo decodes into, not just the value KryoGoldenCompatTest could not have caught the defect it was written to catch, for two independent reasons, and both were found the hard way -- by the defect reaching a running instance. Its ten golden values were Java collections. What OBP-API memoizes is Scala collections; java.util.ArrayList appears in none of the providers. And it compared with ==, under which an empty Scala List EQUALS an empty Queue -- both are Seq, and Seq equality is element-wise. So an empty List written by chill 0.9.3, decoding under 0.9.5 as a scala.collection.immutable.Queue, passed a test whose whole purpose was to notice exactly that, while every call site declaring List failed with ClassCastException. Adds kryo_scala_golden_chill_0_9_3.tsv: eleven Scala values encoded on the pre-migration 2.12 classpath, each recorded with the runtime class it was written as. That third column is the point -- the assertion is now on the class, because the class is what a call site depends on. Force-added past .gitignore for the same reason as the first fixture: it cannot be regenerated once every checkout carries 0.9.5. Subclassing is not drift. A Vector read back as Vector1 is assignable to every signature that named Vector and nothing can tell, so the check is isInstance rather than name equality. The Nil-to-Queue drift is recorded in a knownDrift baseline rather than left red. It is a property of two third-party libraries, not something this branch changes, and the mitigation is elsewhere and tested: Redis.serializationNamespace means a 2.13 instance cannot address a 2.12 entry at all. A permanently red suite is one people learn to ignore; a baseline entry costs a written reason. Anything not listed still fails, and a listed name that stops drifting also fails -- a baseline that outlives its hazard reads as still true to whoever comes next. Run: java 9/10 decode correctly, scala 9/11 decode into an assignable class, scala-list-empty and scala-option-none cold on rollout. --- .../kryo_scala_golden_chill_0_9_3.tsv | 12 ++ .../code/api/cache/KryoGoldenCompatTest.scala | 180 ++++++++++++++---- 2 files changed, 150 insertions(+), 42 deletions(-) create mode 100644 obp-api/src/test/resources/kryo_scala_golden_chill_0_9_3.tsv diff --git a/obp-api/src/test/resources/kryo_scala_golden_chill_0_9_3.tsv b/obp-api/src/test/resources/kryo_scala_golden_chill_0_9_3.tsv new file mode 100644 index 0000000000..5af54008cc --- /dev/null +++ b/obp-api/src/test/resources/kryo_scala_golden_chill_0_9_3.tsv @@ -0,0 +1,12 @@ +# name base64(chill 0.9.3 bytes) runtime class as written +scala-list-string dwEDAwGCYQMBgmIDAYJj scala.collection.immutable.$colon$colon +scala-list-empty dgEA scala.collection.immutable.Nil$ +scala-list-int dwEDAgICBAIG scala.collection.immutable.$colon$colon +scala-vector FAECAwGCeAMBgnk= scala.collection.immutable.Vector +scala-map GwECJwEDAWuxAwF2sScBAwFrsgMBdrI= scala.collection.immutable.Map$Map2 +scala-set FgECAwGCYQMBgmI= scala.collection.immutable.Set$Set2 +scala-seq dwECAwFzsQMBc7I= scala.collection.immutable.$colon$colon +scala-option-some EQEDAWhlcuU= scala.Some +scala-option-none dAE= scala.None$ +scala-tuple JwEDAYJhAgI= scala.Tuple2 +scala-nested-list dwECdwEBAwGCeHcBAgMBgnkDAYJ6 scala.collection.immutable.$colon$colon diff --git a/obp-api/src/test/scala/code/api/cache/KryoGoldenCompatTest.scala b/obp-api/src/test/scala/code/api/cache/KryoGoldenCompatTest.scala index f90d307dcb..cb5b672f02 100644 --- a/obp-api/src/test/scala/code/api/cache/KryoGoldenCompatTest.scala +++ b/obp-api/src/test/scala/code/api/cache/KryoGoldenCompatTest.scala @@ -16,53 +16,76 @@ import scala.util.Try * decode after the rollout. A test environment never shows this, because it starts from an * empty cache." * - * That last sentence is the problem this file exists for. The only Kryo test in the suite, + * That last sentence is the problem this file exists for. The only other Kryo test in the suite, * RedisDeserializeMissTest, round-trips through `encode` and `decode` -- BOTH of which run on * whichever chill is on the classpath. A format change is invisible to it by construction: the * new encoder and the new decoder agree with each other no matter what they agree on. * - * So the fixture had to be produced from outside the current build: - * `src/test/resources/kryo_golden_chill_0_9_3.txt` holds ten values encoded by the real chill - * 0.9.3, taken from the pre-migration worktree's own classpath. It cannot be regenerated once - * this branch merges -- every classpath in the repository will have the new chill, and "what the - * old one wrote" stops being observable. Regenerating it with 0.9.5 would turn this file into a - * test that the new chill can read itself, which is what RedisDeserializeMissTest already does. + * So the fixtures had to be produced from outside the current build. Two of them, both encoded on + * the pre-migration 2.12 classpath by the real chill 0.9.3, neither regenerable once every + * checkout carries 0.9.5: * - * ── What is asserted, and what deliberately is not ── + * kryo_golden_chill_0_9_3.txt ten Java values namebase64 + * kryo_scala_golden_chill_0_9_3.tsv eleven Scala values namebase64runtime class * - * NOT asserted: that every value still decodes. The migration commit accepts that some will not, - * and the accepted consequence is a cold cache -- the codec reports the failure, scalacache - * treats the read as a miss, the source block recomputes and rewrites the key. + * ── Why the second fixture, and why it records a class ── * - * Asserted: that a value either decodes to the SAME value, or fails outright. The outcome that - * must never happen is the third one -- decoding "successfully" into something different. - * A silent misread is not a cold cache; it is wrong data served for a full TTL, and no log line - * anywhere would say so. + * The first version of this file held Java collections only and compared with `==`. It could not + * have caught the defect it was written to catch, for two independent reasons, and both were + * found the hard way -- by the defect reaching a running instance. * - * The count is reported rather than bounded. How many of the ten survive is a fact about two - * library versions, not something this branch controls, and an assertion on it would be a - * number nobody could act on. What the run is for is the third outcome, and the printed - * breakdown, which tells whoever reads it how cold the cache will actually be on rollout. + * What OBP-API actually caches is Scala collections; `java.util.ArrayList` appears nowhere in + * the memoized providers. And under `==` a Scala `List()` EQUALS a `Queue()`: both are `Seq`, and + * Seq equality is element-wise, so an empty one of each compares equal. An empty `List` written + * by 0.9.3 decodes under 0.9.5 into a `scala.collection.immutable.Queue`, which the old assertion + * would have waved through -- while every call site whose signature says `List` fails with + * + * class scala.collection.immutable.Queue cannot be cast to + * class scala.collection.immutable.List + * + * Measured on GET /management/dynamic-message-docs and GET /management/connector-methods: 200 on + * 2.12, 500 on 2.13 reading 2.12's entry, for the whole TTL, because a read that throws does not + * evict the key. + * + * So the Scala fixture records the runtime class each value was written as, and this file asserts + * on THAT. Equality is not enough; the class is what the call site depends on. + * + * ── The four outcomes ── + * + * decodes to the same value, same class fine + * fails to decode fine -- a cold cache, which the upgrade note accepts + * decodes to a DIFFERENT value asserted against from the start + * decodes to an equal value of another CLASS the one that got through, now asserted + * + * Survival counts are reported rather than bounded: how many of the values survive is a fact + * about two third-party libraries, not something this branch controls, and an assertion on it + * would freeze a number nobody could act on. What the run is for is the last two outcomes. */ class KryoGoldenCompatTest extends FlatSpec with Matchers { - private val FIXTURE = "/kryo_golden_chill_0_9_3.txt" + private val JAVA_FIXTURE = "/kryo_golden_chill_0_9_3.txt" + private val SCALA_FIXTURE = "/kryo_scala_golden_chill_0_9_3.tsv" - /** name -> the bytes chill 0.9.3 produced for it. */ - private lazy val golden: List[(String, Array[Byte])] = { - val stream = getClass.getResourceAsStream(FIXTURE) + private def readFixture(path: String): List[Array[String]] = { + val stream = getClass.getResourceAsStream(path) stream should not be null val src = Source.fromInputStream(stream, "UTF-8") try src.getLines() .filterNot(l => l.trim.isEmpty || l.startsWith("#")) - .map { line => - val Array(name, b64) = line.split("\t", 2) - name -> Base64.getDecoder.decode(b64) - }.toList + .map(_.split("\t")) + .toList finally src.close() } - /** The values those bytes are supposed to mean. Written out here, not derived. */ + /** name -> the bytes chill 0.9.3 produced for it. */ + private lazy val javaGolden: List[(String, Array[Byte])] = + readFixture(JAVA_FIXTURE).map(f => f(0) -> Base64.getDecoder.decode(f(1))) + + /** name -> (bytes, the runtime class the value had WHEN WRITTEN). */ + private lazy val scalaGolden: List[(String, Array[Byte], String)] = + readFixture(SCALA_FIXTURE).map(f => (f(0), Base64.getDecoder.decode(f(1)), f(2))) + + /** The values the Java bytes are supposed to mean. Written out here, not derived. */ private val expected: Map[String, Any] = Map( "string" -> "a-cached-string", "int" -> 42, @@ -83,19 +106,90 @@ class KryoGoldenCompatTest extends FlatSpec with Matchers { case (x, y) => x == y } - "the fixture" should "be present and non-trivial" in { + // ── fixtures present ─────────────────────────────────────────────────────────────── + + "both fixtures" should "be present and non-trivial" in { // A fixture that failed to load would make every assertion below vacuous. withClue("kryo_golden_chill_0_9_3.txt is missing or empty -- without it this file asserts " + "nothing at all. It cannot be regenerated from this branch; recover it from git. ") { - golden.size should be >= 8 + javaGolden.size should be >= 8 + } + withClue("kryo_scala_golden_chill_0_9_3.tsv is missing or empty. This is the fixture that " + + "covers what OBP-API actually caches; without it the Java values alone would pass " + + "while the Scala ones drift, which is exactly what happened once already. ") { + scalaGolden.size should be >= 8 + } + scalaGolden.map(_._1).toSet should contain allOf ("scala-list-empty", "scala-map", "scala-option-none") + } + + // ── the assertion the first version was missing ──────────────────────────────────── + + /** + * Class drift that is known, and the reason it can no longer reach a caller. + * + * A signed-off baseline rather than a hard zero, for the same reason the contract suite keeps + * pr90-base.accepted.json: this is a property of two third-party libraries, not something this + * branch can change, and a permanently red suite is a suite people learn to ignore. What must + * stay red is drift that nobody has looked at -- so anything NOT listed here fails, and adding + * a line means writing down why it is safe. + */ + private val knownDrift: Map[String, String] = Map( + "scala-list-empty" -> + ("Nil$ decodes as Queue under chill 0.9.5. Mitigated by Redis.serializationNamespace: the " + + "cache key carries the Scala binary version, so a 2.13 instance cannot address the entry " + + "a 2.12 instance wrote and it expires on its own TTL. CacheSerializationNamespaceTest " + + "pins that isolation; remove it and this becomes reachable again.") + ) + + it should "never decode a Scala value into a DIFFERENT runtime class, except where recorded" in { + import com.twitter.chill.KryoInjection + + val drifted = scalaGolden.flatMap { case (name, bytes, writtenAs) => + KryoInjection.invert(bytes).toOption.flatMap { v => + val nowIs = if (v == null) "null" else v.getClass.getName + // Subclassing is not drift: a Vector written as `Vector` and read back as `Vector1` is + // still assignable to every signature that named Vector, and nothing at a call site can + // tell. What breaks is a class that is merely EQUAL -- List() == Queue() is true, and a + // `List` signature still throws ClassCastException on it. + val assignable = + try Class.forName(writtenAs).isInstance(v) catch { case _: Throwable => nowIs == writtenAs } + if (assignable) None + else Some(s"$name: written as <$writtenAs>, decodes under this chill as <$nowIs>" + + (if (v.isInstanceOf[Iterable[_]]) " -- equal by value, so an == comparison " + + "would call this correct while every call site declaring the original type " + + "fails with ClassCastException" else "")) + } + } + + val unexplained = drifted.filterNot(line => knownDrift.keys.exists(k => line.startsWith(k + ":"))) + drifted.foreach { line => + knownDrift.collectFirst { case (k, why) if line.startsWith(k + ":") => + info(s"known drift -- $line") + info(s" mitigation: $why") + } + } + + withClue(s"${unexplained.size} Scala value(s) drift into a class the original signature cannot " + + s"hold, and are not in knownDrift. This is not a cold cache -- the read SUCCEEDS and " + + s"the caller gets a ClassCastException for the whole TTL, with nothing in any log to " + + s"say so. Either mitigate it or add it to knownDrift with the reason it cannot reach " + + s"a caller:\n${unexplained.mkString("\n")}\n") { + unexplained shouldBe empty + } + + // The baseline must not outlive what it describes. A name listed here that no longer drifts + // is a line nobody will delete, and the next reader takes it as still true. + val staleEntries = knownDrift.keys.filterNot(k => drifted.exists(_.startsWith(k + ":"))).toList + withClue(s"knownDrift lists ${staleEntries.mkString(", ")}, which no longer drift. Remove " + + s"them, or the baseline documents a hazard that stopped existing. ") { + staleEntries shouldBe empty } - golden.map(_._1).toSet should contain allOf ("string", "jlist-string", "jmap") } it should "never decode old bytes into a DIFFERENT value" in { import com.twitter.chill.KryoInjection - val misread = golden.flatMap { case (name, bytes) => + val misread = javaGolden.flatMap { case (name, bytes) => KryoInjection.invert(bytes) match { case scala.util.Success(v) if !sameValue(v, expected(name)) => Some(s"$name: old bytes decoded to <$v> (${v.getClass.getName}) but were written as " + @@ -104,10 +198,8 @@ class KryoGoldenCompatTest extends FlatSpec with Matchers { } } - withClue(s"${misread.size} value(s) written by chill 0.9.3 decode under 0.9.5 into something " + - s"OTHER than what was written. This is the one outcome the upgrade note does not " + - s"cover: not a cold cache, but wrong data served for a full TTL, with nothing in " + - s"any log to say so:\n${misread.mkString("\n")}\n") { + withClue(s"${misread.size} value(s) written by chill 0.9.3 decode under this chill into " + + s"something OTHER than what was written:\n${misread.mkString("\n")}\n") { misread shouldBe empty } } @@ -115,16 +207,20 @@ class KryoGoldenCompatTest extends FlatSpec with Matchers { it should "report how much of an existing cache survives the upgrade" in { import com.twitter.chill.KryoInjection - val (ok, failed) = golden.partition { case (name, bytes) => + val (jOk, jFailed) = javaGolden.partition { case (name, bytes) => Try(KryoInjection.invert(bytes)).toOption.flatMap(_.toOption).exists(sameValue(_, expected(name))) } - // Informational on purpose -- see the header. The rollout consequence of the failures is a + val (sOk, sFailed) = scalaGolden.partition { case (_, bytes, writtenAs) => + KryoInjection.invert(bytes).toOption.exists(v => + try Class.forName(writtenAs).isInstance(v) catch { case _: Throwable => false }) + } + // Informational on purpose -- see the header. The rollout consequence of a failure is a // recompute, which is a cost rather than a defect, and pinning the number would freeze a // property of two third-party libraries. - info(s"${ok.size}/${golden.size} values written by chill 0.9.3 still decode correctly under " + - s"the chill on this classpath") - if (failed.nonEmpty) - info(s"cold on rollout (recomputed on first read): ${failed.map(_._1).mkString(", ")}") + info(s"java ${jOk.size}/${javaGolden.size} values written by chill 0.9.3 still decode correctly") + info(s"scala ${sOk.size}/${scalaGolden.size} values still decode into an assignable class") + if (jFailed.nonEmpty) info(s"cold on rollout (java): ${jFailed.map(_._1).mkString(", ")}") + if (sFailed.nonEmpty) info(s"cold on rollout (scala): ${sFailed.map(_._1).mkString(", ")}") succeed } } From 3d2f10c44946c413aadc700ab1e13a4de828171f Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 25 Aug 2026 23:15:45 +0200 Subject: [PATCH 10/59] test: make DynamicUtilTest drive the production code, and let CI refuse its skips Two ways this suite reported green over things it was not checking. It held a character-for-character copy of the expression that builds the dependency-whitelist source, rather than calling it. The two only stayed in step because whoever edited one happened to see the other -- and this is the one compile that happens reflectively at boot, so a divergence would not have failed at compile time either. The expression is now DynamicUtil.Validation.dependenciesScalaCode, named so both callers can reach it, and the test calls that. Three sandbox scenarios have been cancelling on every run, everywhere, since the build moved to a JDK past 17: SecurityManager enforcement was removed by JEP 411 and finished off by JEP 486, so DynamicUtil.Sandbox is a no-op and `assume` cancels. A cancelled check reads as a passing one in the summary line people actually look at, which is how "canceled 0" and three unrun scenarios coexisted. What goes unchecked is not incidental -- the sandbox is the only thing covering what runtime-compiled endpoint code may touch. OBP_TEST_SANDBOX_REQUIRED=true turns the cancellation into a failure, the same lever RedisTestTarget gives the Redis-dependent checks. Verified both ways: unset gives succeeded 6 / canceled 3, set gives succeeded 6 / failed 3 with a message naming what cannot run and why. The skip still stands by default, because no JDK on this build can enforce. The difference is that it is now somebody's decision rather than a silence. --- .../scala/code/api/util/DynamicUtil.scala | 26 +++++++++--- .../scala/code/util/DynamicUtilTest.scala | 41 +++++++++++++++++-- 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 69ca8dea3c..862bf609c8 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -360,6 +360,26 @@ object DynamicUtil extends MdcLoggable{ object Validation { + /** + * Turn the `dynamic_code_compile_validate_dependencies` props value into the Scala source + * that, once compiled, yields the whitelist. + * + * A named function rather than an inline expression so a test can drive the real thing. + * DynamicUtilTest used to hold a character-for-character copy of it, which meant the two + * could diverge with the test still green -- the copy was only kept in step here because + * whoever edited one happened to see the other. This is the only compile that happens + * reflectively at boot, so nothing at compile time would have caught the divergence either. + * + * `Map[String, String](` rather than `Map(`: the props default is an empty list, and a bare + * `Map()` leaves its type parameters undetermined, so the trailing `.toMap` cannot prove the + * elements are pairs and the reflective compilation fails. The `.toMap` is itself needed + * because `mapValues` returns a view rather than a Map on 2.13. + */ + def dependenciesScalaCode(dependenciesString: String): String = + s"${DynamicUtil.importStatements}" + + dependenciesString.replaceFirst("\\[", "Map[String, String](").dropRight(1) + + ").mapValues(v => StringUtils.split(v, ',').map(_.trim).toSet).toMap" + val dynamicCodeSandboxPermissions = APIUtil.getPropsValue("dynamic_code_sandbox_permissions", "[]").trim val scalaCodePermissioins = "List[java.security.Permission]"+dynamicCodeSandboxPermissions.replaceFirst("\\[","(").dropRight(1)+")" val permissions:Box[List[java.security.Permission]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodePermissioins) @@ -385,11 +405,7 @@ object DynamicUtil extends MdcLoggable{ val allowedRuntimePermissions = permissions.openOrThrowException("Can not compile the props `dynamic_code_sandbox_permissions` to permissions") val dependenciesString = APIUtil.getPropsValue("dynamic_code_compile_validate_dependencies", "[]").trim - // `Map[String, String](` rather than `Map(`: the props default is an empty list, and a bare - // `Map()` leaves its type parameters undetermined, so the trailing .toMap cannot prove the - // elements are pairs and the reflective compilation fails. The .toMap itself is needed because - // mapValues returns a view rather than a Map. - val scalaCodeDependencies = s"${DynamicUtil.importStatements}"+dependenciesString.replaceFirst("\\[","Map[String, String](").dropRight(1) +").mapValues(v => StringUtils.split(v, ',').map(_.trim).toSet).toMap" + val scalaCodeDependencies = dependenciesScalaCode(dependenciesString) val dependenciesBox: Box[Map[String, Set[String]]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodeDependencies) /** diff --git a/obp-api/src/test/scala/code/util/DynamicUtilTest.scala b/obp-api/src/test/scala/code/util/DynamicUtilTest.scala index c1bfd22e2b..c9be8d2452 100644 --- a/obp-api/src/test/scala/code/util/DynamicUtilTest.scala +++ b/obp-api/src/test/scala/code/util/DynamicUtilTest.scala @@ -48,6 +48,35 @@ class DynamicUtilTest extends FlatSpec with Matchers { private val securityManagerUnavailable = "SecurityManager enforcement is not available on JDK 17+ (JEP 411); skip on JDK 21" + /** + * Skip the sandbox checks when no SecurityManager can enforce them -- but let CI refuse the skip. + * + * `assume` cancels, and a cancelled check is indistinguishable from a passing one in every + * report anybody reads: this suite has shown "canceled 0" while three of its scenarios never + * ran, because ScalaTest counts a cancellation separately from a failure and the summary line + * people look at is the failure count. Since JDK 17 removed SecurityManager enforcement + * (JEP 411, completed by JEP 486), DynamicUtil.Sandbox is a no-op on any modern JDK and these + * three have been skipping on every run, everywhere, for as long as the build has been on 21+. + * + * That is a real gap, not a formality: the sandbox is what stops runtime-compiled endpoint code + * from reading the filesystem or opening sockets, and nothing else covers it. + * + * OBP_TEST_SANDBOX_REQUIRED=true turns the cancellation into a failure, the same lever + * RedisTestTarget gives the Redis-dependent checks. Set it wherever a JDK that can still + * enforce is available; leave it unset and the skip stands, but now it is a decision somebody + * made rather than a silence. + */ + private def requireSecurityManager(): Unit = + if (System.getSecurityManager == null) { + val required = sys.env.get("OBP_TEST_SANDBOX_REQUIRED").exists(_.trim.equalsIgnoreCase("true")) + if (required) + fail("OBP_TEST_SANDBOX_REQUIRED=true but no SecurityManager is installed, so the sandbox " + + "checks cannot run. They are the only cover for what runtime-compiled endpoint code " + + "is allowed to touch -- run them on a JDK that still enforces, or unset the variable " + + "to go back to skipping.") + else cancel(securityManagerUnavailable) + } + implicit val formats = code.api.util.CustomJsonFormats.formats @@ -115,14 +144,18 @@ class DynamicUtilTest extends FlatSpec with Matchers { val dependenciesString = """[NewStyle.function.getClass.getTypeName -> "*",CompiledObjects.getClass.getTypeName -> "sandbox",HttpCode.getClass.getTypeName -> "200",DynamicCompileEndpoint.getClass.getTypeName -> "getPathParams, scalaFutureToBoxedJsonResponse",APIUtil.getClass.getTypeName -> "errorJsonResponse, errorJsonResponse$default$1, errorJsonResponse$default$2, errorJsonResponse$default$3, errorJsonResponse$default$4, scalaFutureToLaFuture, futureToBoxedResponse",ErrorMessages.getClass.getTypeName -> "*",ExecutionContext.Implicits.getClass.getTypeName -> "global",JSONFactory400.getClass.getTypeName -> "createBanksJson",classOf[Sandbox].getTypeName -> "runInSandbox",classOf[CallContext].getTypeName -> "*",classOf[ResourceDoc].getTypeName -> "getPathParams","scala.reflect.runtime.package$" -> "universe",PractiseEndpoint.getClass.getTypeName + "*" -> "*"]""".stripMargin - val scalaCode2 = s"${DynamicUtil.importStatements}"+dependenciesString.replaceFirst("\\[","Map[String, String](").dropRight(1) +").mapValues(v => StringUtils.split(v, ',').map(_.trim).toSet).toMap" + // DynamicUtil.Validation.dependenciesScalaCode, not a copy of it. This line used to be a + // character-for-character duplicate of the production expression, which meant an edit to + // either one left the test green while the two disagreed -- and this is the only compile that + // happens reflectively at boot, so nothing at compile time would have noticed either. + val scalaCode2 = DynamicUtil.Validation.dependenciesScalaCode(dependenciesString) val dependenciesBox2: Box[Map[String, Set[String]]] = DynamicUtil.compileScalaCode(scalaCode2) val dependencies2 = dependenciesBox2.openOrThrowException("Can not compile the string to Map") dependencies2.toString contains ("code.api.util.NewStyle") shouldBe (true) } "Sandbox.createSandbox method" should "should throw exception" taggedAs DynamicUtilsTag in { - assume(System.getSecurityManager != null, securityManagerUnavailable) + requireSecurityManager() val permissionList = List( // new java.net.SocketPermission("ir.dcs.gla.ac.uk:80","connect,resolve"), ) @@ -147,7 +180,7 @@ class DynamicUtilTest extends FlatSpec with Matchers { } "Sandbox.sandbox method test bankId" should "should throw exception" taggedAs DynamicUtilsTag in { - assume(System.getSecurityManager != null, securityManagerUnavailable) + requireSecurityManager() intercept[AccessControlException] { Sandbox.sandbox(bankId= "abc").runInSandbox { BankId("123" ) @@ -162,7 +195,7 @@ class DynamicUtilTest extends FlatSpec with Matchers { } "Sandbox.sandbox method test default permission" should "should throw exception" taggedAs DynamicUtilsTag in { - assume(System.getSecurityManager != null, securityManagerUnavailable) + requireSecurityManager() intercept[AccessControlException] { Sandbox.sandbox(bankId= "abc").runInSandbox { scala.io.Source.fromURL("https://apisandbox.openbankproject.com/") From 68e997261f1f51e7b5281be9608a975dce793c64 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 26 Aug 2026 02:04:09 +0200 Subject: [PATCH 11/59] fix: stop reporting six client-side conditions as server faults A 5xx tells a caller the server broke and the request is worth retrying. Each of these is a client-side condition, so the retry can never succeed -- and one of them sits on a payment path, where that loop is the expensive kind. getUserInvitation secretLink.toLong threw NumberFormatException on any non-numeric segment -> 400 (InvalidNumber, which already existed and says exactly this) createConsumerDynamicRegistration verifyJwt does not return false for a missing or unparseable PSD2-CERT, it THROWS ("No PEM-encoded keys found"), and booleanToFuture only guards the false case -> 400 getSignalChannelInfo a plain RuntimeException for "no such channel" -> 404, with a new SignalChannelNotFound (OBP-39021, continuing the Signal series) createTransactionRequest four raw throws -- not authenticated, no such bank, no such account, no such view -> 401/404 createTestEmail two "server is not configured" checks answered 500. The server is not broken, it is unconfigured, and neither resolves without an operator editing props -> 503 getConnectorMethodNames is different in kind: it is a regression this branch's own base introduced, and it is fixed at the source rather than at the endpoint. java.lang.reflect.Proxy passes null for a method declaring no parameters; cglib, which the ByteBuddy swap replaced, passed a zero-length array. Everything downstream treats args as a collection -- `.zip(args)`, `args.collectFirst`, `extractKeyParams(args)`. isInheritedMember covers members Connector does not declare, but a NO-ARGUMENT method that Connector DOES declare slips past it and lands in routeToConnector. `callableMethods` is exactly that. Confirmed by running both builds with the same grants against the same request: GET /obp/v6.0.0/system/connector-method-names answers 200 on the 2.12/cglib build and 500 on this one, with `Cannot invoke "scala.collection.IterableOnce.knownSize()" because "that" is null` -- which is `zip` being handed the null. Normalising args at the handler entry restores what cglib did, which is what a toolchain migration owes its callers, and closes the whole family rather than this one endpoint. Sweep after these: 57 tests, 0 failures, 0 suites aborted (was 47/9). --- .../scala/code/api/util/ErrorMessages.scala | 1 + .../scala/code/api/v4_0_0/Http4s400.scala | 28 +++++++++++++++---- .../scala/code/api/v5_1_0/Http4s510.scala | 9 +++++- .../scala/code/api/v6_0_0/Http4s600.scala | 8 +++++- .../scala/code/api/v7_0_0/Http4s700.scala | 7 +++-- .../scala/code/bankconnectors/package.scala | 18 +++++++++++- 6 files changed, 61 insertions(+), 10 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala index 33040e83d4..f35a35fe31 100644 --- a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala +++ b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala @@ -828,6 +828,7 @@ object ErrorMessages { val ChatMessageTypeNotAllowed = "OBP-39018: Invalid message_type. Allowed values: text, system." val SignalMessageTooLong = "OBP-39019: Signal message exceeds the maximum allowed length." val SignalMessageContainsDangerousCharacters = "OBP-39020: Signal message contains control or bidirectional-override characters, which are not allowed." + val SignalChannelNotFound = "OBP-39021: Signal Channel not found." // Transaction Request related messages (OBP-40XXX) val InvalidTransactionRequestType = "OBP-40001: Invalid value for TRANSACTION_REQUEST_TYPE" diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala index d6c1e7c8ef..eb6d64e4fa 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala @@ -3042,15 +3042,26 @@ object Http4s400 { EndpointHelpers.executeFutureCreated(req) { val bodyStr = cc.httpBody.getOrElse("") for { - user <- Future { cc.user.openOrThrowException(AuthenticatedUserIsRequired) } - bank <- Future { cc.bank.getOrElse(throw new RuntimeException(BankNotFound)) } - account <- Future { cc.bankAccount.getOrElse(throw new RuntimeException(BankAccountNotFound)) } + // These four used to throw raw exceptions, which the converter can only render as + // OBP-50000 / HTTP 500. Every one of them is a client-side condition -- not + // authenticated, no such bank, no such account, no such view -- and a 500 tells a + // caller with retry logic to keep sending a request that cannot succeed. This is a + // payment path, so that retry loop is the expensive kind. + user <- NewStyle.function.tryons(AuthenticatedUserIsRequired, 401, Some(cc)) { + cc.user.openOrThrowException(AuthenticatedUserIsRequired) + } + bank <- NewStyle.function.tryons(BankNotFound, 404, Some(cc)) { + cc.bank.getOrElse(throw new NoSuchElementException(bankIdStr)) + } + account <- NewStyle.function.tryons(BankAccountNotFound, 404, Some(cc)) { + cc.bankAccount.getOrElse(throw new NoSuchElementException(accountIdStr)) + } json <- NewStyle.function.tryons( s"$InvalidJsonFormat Empty or invalid request body.", 400, Some(cc)) { com.openbankproject.commons.util.JsonAliases.parse(bodyStr) } transactionRequestType = TransactionRequestType(transactionRequestTypeStr) - view <- Future { + view <- NewStyle.function.tryons(s"$ViewNotFound Current view_id($viewIdStr)", 404, Some(cc)) { // System views (owner, accountant, etc.) and custom views (e.g. VRP // `_vrp-…` views) are stored separately. Try system first; fall back // to the account-scoped custom view. SS.init only needs *some* View @@ -6393,7 +6404,14 @@ object Http4s400 { case req @ GET -> `prefixPath` / "banks" / _ / "user-invitations" / secretLink => EndpointHelpers.withUserAndBank(req) { (_, bank, cc) => for { - (invitation, _) <- NewStyle.function.getUserInvitation(bank.bankId, secretLink.toLong, Some(cc)) + // `secretLink.toLong` used to run unguarded, so any non-numeric path segment left a + // NumberFormatException to escape as OBP-50000 / HTTP 500 -- a malformed identifier + // reported to the caller as a server fault, which tells a client with retry logic to + // keep sending a request that can never succeed. + secret <- NewStyle.function.tryons(s"$InvalidNumber Invalid SECRET_LINK: it must be a number.", 400, Some(cc)) { + secretLink.toLong + } + (invitation, _) <- NewStyle.function.getUserInvitation(bank.bankId, secret, Some(cc)) } yield JSONFactory400.createUserInvitationJson(invitation) } } diff --git a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala index 80d5e2b17a..c7f6ea11cb 100644 --- a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala +++ b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala @@ -3050,9 +3050,16 @@ object Http4s510 { com.openbankproject.commons.util.JsonAliases.parse(cc.httpBody.getOrElse("")).extract[ConsumerJwtPostJsonV510] } pem = APIUtil.`getPSD2-CERT`(cc.requestHeaders) - _ <- Helper.booleanToFuture(PostJsonIsNotSigned, 400, Some(cc)) { + // `verifyJwt` does not merely return false for a bad certificate -- it THROWS + // ("No PEM-encoded keys found") when the PSD2-CERT header is absent or unparseable, + // and booleanToFuture only guards the false case, so the exception escaped as + // OBP-50000 / HTTP 500. A missing or malformed client certificate is a client error; + // reporting it as a server fault tells a caller with retry logic to keep sending a + // request that cannot ever succeed. + signatureValid <- NewStyle.function.tryons(PostJsonIsNotSigned, 400, Some(cc)) { JwtUtil.verifyJwt(postedJwt.jwt, pem.getOrElse("")) } + _ <- Helper.booleanToFuture(PostJsonIsNotSigned, 400, Some(cc)) { signatureValid } postedJson <- NewStyle.function.tryons(InvalidJsonFormat, 400, Some(cc)) { com.openbankproject.commons.util.JsonAliases.parse(JwtUtil.getSignedPayloadAsJson(postedJwt.jwt).getOrElse("{}")).extract[ConsumerPostJsonV510] } diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index 4927967ba2..b4559f7cf2 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -2621,9 +2621,15 @@ object Http4s600 { code.api.cache.RedisMessaging.validateChannelName(channelName) } info <- Future(code.api.cache.RedisMessaging.channelInfo(channelName)) + // A plain RuntimeException here surfaced as OBP-50000 / HTTP 500. "The thing you asked + // for does not exist" is the textbook 404; a 500 says the server broke, and a client + // cannot tell from it that retrying is pointless. (count, ttl) <- info match { case Some((c, t)) => Future.successful((c, t)) - case None => Future.failed(new RuntimeException(s"Channel '$channelName' not found")) + case None => + NewStyle.function.tryons(s"$SignalChannelNotFound Channel '$channelName' not found.", 404, Some(cc)) { + throw new NoSuchElementException(channelName) + } } } yield SignalChannelInfoJsonV600(channelName, count, ttl) } diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala index 7486daecb2..f80101b1b6 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala @@ -1953,14 +1953,17 @@ object Http4s700 { _ <- Helper.booleanToFuture(UserEmailAddressMissing, 400, Some(cc)) { toAddress.nonEmpty } + // 503, not 500. The server is not broken -- it is not configured to do this, and a + // 500 tells a caller with retry logic that the fault is transient. Neither of these + // resolves without an operator editing props. _ <- Helper.booleanToFuture( s"$IncompleteServerConfiguration portal_external_url is not set — signup-validation and password-reset emails will not be delivered.", - 500, Some(cc)) { + 503, Some(cc)) { portalUrlBox.isDefined } _ <- Helper.booleanToFuture( s"$IncompleteServerConfiguration mail.users.userinfo.sender.address is still the default 'noreply@example.com' — most SMTP servers will reject this From address.", - 500, Some(cc)) { + 503, Some(cc)) { fromAddress != "noreply@example.com" } sendOutcome <- Future { diff --git a/obp-api/src/main/scala/code/bankconnectors/package.scala b/obp-api/src/main/scala/code/bankconnectors/package.scala index 42b76021da..1211117536 100644 --- a/obp-api/src/main/scala/code/bankconnectors/package.scala +++ b/obp-api/src/main/scala/code/bankconnectors/package.scala @@ -136,7 +136,23 @@ package object bankconnectors extends MdcLoggable { } val intercept: InvocationHandler = new InvocationHandler { - override def invoke(proxy: AnyRef, method: Method, args: Array[AnyRef]): AnyRef = { + override def invoke(proxy: AnyRef, method: Method, rawArgs: Array[AnyRef]): AnyRef = { + // `java.lang.reflect.Proxy` passes null for a method that declares no parameters; cglib, + // which this replaced, passed a zero-length array. Everything downstream treats args as a + // collection -- `.zip(args)`, `args.collectFirst`, `extractKeyParams(args)` -- and every + // one of those throws on null. + // + // isInheritedMember covers the members Connector does not declare, but a NO-ARGUMENT + // method that Connector DOES declare slips past it and lands in routeToConnector. + // Measured on GET /obp/v6.0.0/system/connector-method-names, which reads + // `connector.callableMethods`: 200 on the 2.12/cglib build, 500 on this one, with + // `Cannot invoke "scala.collection.IterableOnce.knownSize()" because "that" is null` -- + // which is `zip` being handed the null. + // + // Normalising to an empty array restores exactly what cglib did, which is what a + // toolchain migration owes its callers. `method.invoke(target, args: _*)` is unaffected: + // it compiles to Java varargs and an empty array means the same as null there. + val args: Array[AnyRef] = if (rawArgs == null) Array.empty[AnyRef] else rawArgs if (method.getReturnType.getName == "scala.concurrent.Future" && !canOpenFuture(method.getName)) { throw new RuntimeException(ServiceIsTooBusy + s"Current Service(${method.getName})") } else if (method.getName.contains("$default$") || ConnectorProxy.isInheritedMember(method)) { From dcd5b329a91c6c0d2393048e63134df4b4ae0ef9 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 26 Aug 2026 02:04:30 +0200 Subject: [PATCH 12/59] test: fix two ways the sweep reported the wrong thing, and record the deviations Three of the sweep's findings turned out to be the sweep's own defects, and they are the same defect twice over: a rule copied instead of called. SuccessSweepTest kept its own placeholder rule, and it had drifted from EndpointCatalog's -- the catalog substitutes any segment ending in ID, _CODE or _NAME, this one knew _ID and _CODE and had never learnt _NAME. So /signal/channels/CHANNEL_NAME/info was a placeholder to the catalog, which duly replaced it with a channel that does not exist, and NOT a placeholder here, so this suite selected it as an endpoint needing nothing created first and then failed it for answering 404. The first half of its condition was vacuous as well: `concretePath(doc) == concretePath(doc, Map.empty)` compares a default argument with the same value passed explicitly. Now calls EndpointCatalog.hasPlaceholder. EndpointCatalog resolved VIEW_ID but not GRANT_VIEW_ID, so an endpoint that looks up the view before it checks roles answered on the view and the role gate never ran. That is the same reason a real bank id is already passed for the role assertion. createTransactionRequestFreeForm was reported as "expected 403, got 500" -- two defects stacked, the endpoint's raw throw and this placeholder never resolving. Two deviations remain and are recorded rather than left red, with the reason each is not a defect: verifyRequestSignResponse refuses with OBP-20311, JWS request signing -- a third mechanism that authMode cannot express, so neither the doc nor this sweep can declare it createTransactionRequestFreeForm answers 400 rather than 403; the endpoint deliberately delegates the decision to the connector and says so in its own comment, and an existing test depends on it. Whether an authorisation failure should be 400 at all is a product question The exemption list expires on its own: an entry whose endpoint stops deviating fails the run, and so does one naming an operationId no longer in the catalog. A baseline that outlives its hazard reads as still true to whoever comes next. One process note, because it nearly went the other way. The stale-entry scenario was first written inside the per-version loop, so it registered once per API version and the duplicate name aborted the whole suite at construction. Maven reported "Tests: succeeded 31, failed 0" and BUILD SUCCESS over a suite that ran nothing -- the same shape as the cancelled-check and the zero-test cases already fixed on this branch. Reading `Suites: completed N, aborted 0` alongside the test counts is what caught it. --- .../scala/code/api/sweep/AuthSweepTest.scala | 68 ++++++++++++++++++- .../code/api/sweep/EndpointCatalog.scala | 10 ++- .../code/api/sweep/SuccessSweepTest.scala | 24 ++++--- 3 files changed, 89 insertions(+), 13 deletions(-) diff --git a/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala b/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala index d919ade548..966c6a821c 100644 --- a/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala +++ b/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala @@ -104,12 +104,49 @@ class AuthSweepTest extends ServerSetupWithTestData with DefaultUsers { // ── the three checks, each returning a failure line or None ────────────────── + /** + * Deviations that are deliberate, with the reason each one is not a defect. + * + * A signed-off list rather than a hard zero, for the same reason KryoGoldenCompatTest keeps + * knownDrift: a permanently red suite is one people learn to ignore, and the two entries here + * are both behaviour somebody chose and wrote down. Anything NOT listed still fails, and + * adding a line costs a written justification. + */ + private val expectedAuthDeviation: Map[String, String] = Map( + "OBPv4.0.0-verifyRequestSignResponse" -> + ("Refuses with OBP-20311 'The Request is not signed' -- JWS request signing, a third " + + "authentication mechanism alongside user and application. ResourceDoc has no way to " + + "declare it: authMode covers user/application only, so neither the doc nor this sweep " + + "can express the requirement. The 401 is correct; only the message differs."), + "OBPv4.0.0-createTransactionRequestFreeForm" -> + ("Answers 400 InsufficientAuthorisationToCreateTransactionRequest rather than 403. The " + + "endpoint deliberately does no upfront view/role check and delegates the decision to " + + "checkAuthorisationToCreateTransactionRequest inside the connector -- its own comment " + + "says so, and an existing test depends on it. Whether an authorisation failure ought to " + + "be 400 at all is a product question, not something to change from inside a sweep.") + ) + + /** Which exemptions were actually needed this run -- see the stale-entry scenario below. */ + private val deviationsUsed = java.util.concurrent.ConcurrentHashMap.newKeySet[String]() + + private def deviationFor(doc: ResourceDoc): Option[String] = { + val why = expectedAuthDeviation.get(doc.operationId) + if (why.isDefined) deviationsUsed.add(doc.operationId) + why + } + private def checkAnonymousIs401(doc: ResourceDoc): Option[String] = { val (code, json) = call(doc.requestVerb, EndpointCatalog.concretePath(doc), Map.empty) if (code != 401) Some(s"${describe(doc)} -- expected 401 for an anonymous call, got $code") else if (messageOf(json) != AuthenticatedUserIsRequired) - Some(s"${describe(doc)} -- 401 but message was '${messageOf(json)}', expected '$AuthenticatedUserIsRequired'") + deviationFor(doc) match { + case Some(why) => + info(s"${describe(doc)} -- 401 with '${messageOf(json)}'; expected deviation: $why") + None + case None => + Some(s"${describe(doc)} -- 401 but message was '${messageOf(json)}', expected '$AuthenticatedUserIsRequired'") + } else None } @@ -154,8 +191,14 @@ class AuthSweepTest extends ServerSetupWithTestData with DefaultUsers { val (code, json) = call(doc.requestVerb, path, noRoleHeaders) val roles = doc.roles.getOrElse(Nil).map(_.toString).mkString(",") if (code != 403) - Some(s"${doc.operationId} ${doc.requestVerb} $path -- roles $roles: " + - s"expected 403 for a user holding no entitlements, got $code") + deviationFor(doc) match { + case Some(why) => + info(s"${doc.operationId} answered $code rather than 403; expected deviation: $why") + None + case None => + Some(s"${doc.operationId} ${doc.requestVerb} $path -- roles $roles: " + + s"expected 403 for a user holding no entitlements, got $code") + } else if (!messageOf(json).startsWith(UserHasMissingRoles)) Some(s"${doc.operationId} ${doc.requestVerb} $path -- 403 but message was " + s"'${messageOf(json)}', expected it to start with '$UserHasMissingRoles'") @@ -216,5 +259,24 @@ class AuthSweepTest extends ServerSetupWithTestData with DefaultUsers { } } } + + // Declared after both version loops, so deviationsUsed is complete when it runs. + scenario("no expectedAuthDeviation entry outlives the behaviour it excuses", AuthSweep) { + import scala.jdk.CollectionConverters._ + val used = deviationsUsed.asScala.toSet + val stale = expectedAuthDeviation.keySet -- used + val unknown = expectedAuthDeviation.keySet -- EndpointCatalog.all.map(_.operationId).toSet + + withClue(s"these endpoints no longer deviate, so their exemption is a claim that stopped " + + s"being true and the next reader will take it as still true: ${stale.mkString(", ")}. " + + s"Delete the entry. ") { + stale shouldBe empty + } + withClue(s"these operationIds are not in the catalog at all -- renamed or removed, and " + + s"the exemption was left behind: ${unknown.mkString(", ")}. ") { + unknown shouldBe empty + } + info(s"${used.size} deviation(s) exercised: ${used.mkString(", ")}") + } } } diff --git a/obp-api/src/test/scala/code/api/sweep/EndpointCatalog.scala b/obp-api/src/test/scala/code/api/sweep/EndpointCatalog.scala index 9e1af85e14..6f6eae87eb 100644 --- a/obp-api/src/test/scala/code/api/sweep/EndpointCatalog.scala +++ b/obp-api/src/test/scala/code/api/sweep/EndpointCatalog.scala @@ -86,6 +86,9 @@ object EndpointCatalog { * ACCOUNT are literals, CARD_ID and ACCOUNT_ID are placeholders — and it needs no maintenance * when a new literal appears, because an unrecognised segment is left alone by default. */ + /** True when the URL carries at least one segment concretePath would substitute. */ + def hasPlaceholder(doc: ResourceDoc): Boolean = doc.requestUrl.split("/").exists(isPlaceholder) + private def isPlaceholder(seg: String): Boolean = seg.nonEmpty && seg == seg.toUpperCase && @@ -130,7 +133,12 @@ object EndpointCatalog { /** Well-formed, and deliberately not present. */ private def defaultValue(seg: String): String = seg match { case "ACCOUNT_ID" | "USER_ID" => "00000000-0000-0000-0000-000000000000" - case "VIEW_ID" => "owner" + // GRANT_VIEW_ID alongside VIEW_ID, and for the same reason a real bank id is passed for the + // role assertion: an endpoint that resolves the view before it checks roles answers 404 for a + // view that does not exist, and the role gate never runs. Measured on + // createTransactionRequestFreeForm, which the sweep reported as "expected 403, got 500" -- + // two defects stacked, the endpoint's raw throw AND this placeholder never resolving. + case "VIEW_ID" | "GRANT_VIEW_ID" => "owner" case "USER_EMAIL" => "sweep-no-such-user@example.com" case _ => "sweep-no-such-" + seg.toLowerCase.replace('_', '-') } diff --git a/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala b/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala index 69fc079403..521d5b2be9 100644 --- a/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala +++ b/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala @@ -126,15 +126,21 @@ class SuccessSweepTest extends ServerSetupWithTestData with DefaultUsers { (resp.status.code, json) } - /** No ALL_CAPS placeholder in the URL — nothing to create before calling. */ - private def hasNoPathVariable(doc: ResourceDoc): Boolean = - EndpointCatalog.concretePath(doc) == EndpointCatalog.concretePath(doc, Map.empty) && - !doc.requestUrl.split("/").exists { seg => - seg.nonEmpty && seg == seg.toUpperCase && seg.length > 2 && - seg.forall(c => c.isLetter || c == '_' || c.isDigit) && - (seg.endsWith("_ID") || seg.endsWith("_CODE") || - seg == "PROVIDER" || seg == "USERNAME" || seg == "USER_EMAIL") - } + /** + * No ALL_CAPS placeholder in the URL -- nothing to create before calling. + * + * EndpointCatalog.hasPlaceholder, not a local copy of the rule. This used to hold its own, + * and the two had already drifted: the catalog substitutes any segment ending in ID, _CODE or + * _NAME, this one looked for _ID and _CODE and had never learnt about _NAME. So + * `/signal/channels/CHANNEL_NAME/info` was a placeholder to the catalog -- which duly replaced + * it with a channel that does not exist -- and NOT a placeholder here, so this suite selected + * it as an endpoint that "needs nothing created first" and then failed it for answering 404. + * + * The first half of the old condition was worse than wrong, it was vacuous: + * `concretePath(doc) == concretePath(doc, Map.empty)` compares a default argument with the same + * value passed explicitly, so it is true for every doc and filtered nothing. + */ + private def hasNoPathVariable(doc: ResourceDoc): Boolean = !EndpointCatalog.hasPlaceholder(doc) private lazy val inScope: List[ResourceDoc] = EndpointCatalog.all From 1edd5bdcf7b737b8fbfa449bf956180082cef474 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 26 Aug 2026 02:15:34 +0200 Subject: [PATCH 13/59] test: count skips from what the XML actually contains, not an absent attribute The audit line read `skipped="N"` off . ScalaTest's JUnit reporter does not emit that attribute -- it puts a child inside each cancelled -- so the counter was structurally always zero. A run in which an entire suite cancelled every one of its tests would still have printed "0 skipped/canceled", and that is the number somebody checks precisely when they suspect tests are not running. Measured on the run that prompted this: the audit said 0, the reports held 15 across eight suites. Twelve were news. NginxForwarderTest (4), Http4sServerIntegrationTest, EmbeddedRabbitMQ, two v6 integration suites, RootAndBanksTest (2), BankAccountCreationListenerTest (2) -- integration and infrastructure tests, the category where a silent skip costs the most. Only DynamicUtilTest's three were already known, and only because somebody had looked at them by hand. Counts the child elements, keeping the attribute path first for reporters that do emit it. Why each one skips, and whether it should, needs environment decisions and is not part of this change. Fourth instance of one family on this branch: a cancelled check reporting as a pass, a zero-test run reporting as a pass, an aborted suite reporting BUILD SUCCESS, and now a skip counter that could not count. Each sat in the layer whose job is to notice that tests did not run. --- run_tests_parallel.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/run_tests_parallel.sh b/run_tests_parallel.sh index 1066ab2a6e..2ec56d5b72 100755 --- a/run_tests_parallel.sh +++ b/run_tests_parallel.sh @@ -516,7 +516,17 @@ while IFS= read -r _f; do fi _fa=$(_sf_attr "$_head" failures); _fa=${_fa:-0} _e=$(_sf_attr "$_head" errors); _e=${_e:-0} - _sk=$(_sf_attr "$_head" skipped); _sk=${_sk:-0} + # ScalaTest's JUnit XML reporter does NOT put a skipped="N" attribute on ; + # it emits a child inside each cancelled . Reading the attribute + # therefore always yielded 0, so this line reported "0 skipped/canceled" for a run in + # which DynamicUtilTest cancelled three of its nine -- and would have reported the same + # for a suite that cancelled every one of its tests. That is the number somebody checks + # precisely when they suspect tests are not running, so it has to be counted from what + # is actually in the file. Attribute first for reporters that do emit it, child elements + # otherwise. + _sk=$(_sf_attr "$_head" skipped) + if [[ -z "$_sk" ]]; then _sk=$(grep -c "/dev/null); fi + _sk=${_sk:-0} SF_TOTAL=$((SF_TOTAL+_t)); SF_FAIL=$((SF_FAIL+_fa)); SF_ERR=$((SF_ERR+_e)); SF_SKIP=$((SF_SKIP+_sk)) if [[ $_fa -ne 0 || $_e -ne 0 ]]; then SF_BAD+=("$(basename "$_f" | sed 's/^TEST-//; s/\.xml$//'): $_fa failed, $_e errors") From cea54ffd7505523b32f68a56abb6f8ad0990acc5 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Sun, 16 Aug 2026 09:17:20 +0200 Subject: [PATCH 14/59] fix: drop CallContext from the endpoint-mapping and FX rate cache keys CacheKeyFromArguments renders every parameter that is not annotated @CacheKeyOmit. CallContext carries per-request state (startTime, correlationId, url, verb, ipAddress, user), so both keys were unique per request: the cache could never hit, and getCurrentFxRateCached wrote a fresh Redis entry per call that lived out its TTL. getEndpointMappings additionally cached the (mappings, callContext) tuple. chill/Kryo cannot encode the lambda reachable through CallContext.resourceDocument, so every write failed and cachePut swallowed it as "result served uncached" - endpointMapping.cache.ttl.seconds bought nothing but a WARN per call. A hit would also have handed the caller the originating request's CallContext. Split the memoized half into getEndpointMappingsCached(bankId) rather than annotating callContext on the caller: CacheKeyFromArguments reads the parameters of the method whose body ends in buildCacheKey, so binding the result to a val first leaves it with no parameters and it emits Nil.mkString("_") - an empty argument segment, i.e. every bankId sharing one entry. Verified with javap that the key now renders bankId :: Nil, and that getCurrentFxRateCached renders bankId :: from :: to :: Nil. Add invalidateEndpointMappingCache() on create/update/delete, mirroring invalidateMethodRoutingCache: while callContext was in the key nothing could hit, so a stale entry was unreachable by construction; now that the cache works, writes have to publish themselves. --- .../main/scala/code/api/util/NewStyle.scala | 56 +++++++++++++++++-- .../LocalMappedConnectorInternal.scala | 11 +++- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/NewStyle.scala b/obp-api/src/main/scala/code/api/util/NewStyle.scala index 8145f828b9..08ba3303ed 100644 --- a/obp-api/src/main/scala/code/api/util/NewStyle.scala +++ b/obp-api/src/main/scala/code/api/util/NewStyle.scala @@ -3341,7 +3341,9 @@ object NewStyle extends MdcLoggable{ def createOrUpdateEndpointMapping(bankId: Option[String], endpointMapping: EndpointMappingT, callContext: Option[CallContext]) = { validateBankId(bankId, callContext) Future { - (EndpointMappingProvider.endpointMappingProvider.vend.createOrUpdate(bankId, endpointMapping), callContext) + val result = EndpointMappingProvider.endpointMappingProvider.vend.createOrUpdate(bankId, endpointMapping) + invalidateEndpointMappingCache() + (result, callContext) } map { i => (connectorEmptyResponse(i._1, callContext), i._2) } @@ -3350,12 +3352,31 @@ object NewStyle extends MdcLoggable{ def deleteEndpointMapping(bankId: Option[String], endpointMappingId: String, callContext: Option[CallContext]) = { validateBankId(bankId, callContext) Future { - (EndpointMappingProvider.endpointMappingProvider.vend.delete(bankId, endpointMappingId), callContext) + val result = EndpointMappingProvider.endpointMappingProvider.vend.delete(bankId, endpointMappingId) + invalidateEndpointMappingCache() + (result, callContext) } map { i => (connectorEmptyResponse(i._1, callContext), i._2) } } + /** + * Drop every memoized `getEndpointMappings(...)` entry after a mapping is created, + * updated, or deleted, so the change takes effect on the next request instead of waiting + * out `endpointMapping.cache.ttl.seconds`. Mirrors invalidateMethodRoutingCache: the + * memoize key embeds the literal method name, so one pattern delete clears every bankId + * variant. The pattern is a prefix of the actual name the macro renders + * (`getEndpointMappingsCached`), which is what makes one wildcard cover it. No-op / logged + * when Redis is unavailable (deleteKeysByPattern swallows and returns 0). + * + * This became necessary with the cache key fix below. While callContext was part of the + * key nothing could ever hit, so a stale entry was unreachable by construction; now that + * the cache works, writes have to publish themselves. + */ + private def invalidateEndpointMappingCache(): Unit = { + Redis.deleteKeysByPattern("*getEndpointMappings*") + } + def getEndpointMappingById(bankId: Option[String], endpointMappingId : String, callContext: Option[CallContext]): OBPReturnType[EndpointMappingT] = { validateBankId(bankId, callContext) @@ -3378,18 +3399,41 @@ object NewStyle extends MdcLoggable{ private[this] val endpointMappingTTL = APIUtil.getPropsValue(s"endpointMapping.cache.ttl.seconds", "0").toInt - def getEndpointMappings(bankId: Option[String], callContext: Option[CallContext]): OBPReturnType[List[EndpointMappingT]] = Future{ + /** + * The memoized half of getEndpointMappings, split into its own method for two reasons. + * + * Neither the key nor the cached value may mention the callContext. The key, because + * CacheKeyFromArguments renders every un-annotated parameter and CallContext carries + * per-request state (startTime, correlationId, url, verb, ipAddress, user) - keying on it + * made the key unique per request, so the cache could never hit. The value, because a hit + * would hand the caller the originating request's CallContext, and because chill/Kryo + * cannot encode the lambda reachable through CallContext.resourceDocument: every write of + * the old (mappings, callContext) tuple failed and cachePut swallowed it as "result served + * uncached", so endpointMapping.cache.ttl.seconds bought nothing but a WARN per call. + * + * A parameter-less signature rather than `@CacheKeyOmit callContext` on the caller, because + * CacheKeyFromArguments reads the parameters of the method whose body ENDS in buildCacheKey. + * Binding the result to a val first (`val x = buildCacheKey {...}; (x, callContext)`) leaves + * the macro with no parameters to render and it emits `Nil.mkString("_")` - an empty + * argument segment, i.e. every bankId sharing one entry. Keep buildCacheKey as the tail + * expression here; `NewStyle.function.getEndpointMappings` is verified by javap to render + * `bankId :: Nil`. + */ + private def getEndpointMappingsCached(bankId: Option[String]): List[EndpointMappingT] = { import scala.concurrent.duration._ - validateBankId(bankId, callContext) - var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString) CacheKeyFromArguments.buildCacheKey { Caching.memoizeSyncWithProvider(Some(cacheKey.toString()))(endpointMappingTTL.second) { - {(EndpointMappingProvider.endpointMappingProvider.vend.getAllEndpointMappings(bankId), callContext)} + EndpointMappingProvider.endpointMappingProvider.vend.getAllEndpointMappings(bankId) } } } + + def getEndpointMappings(bankId: Option[String], callContext: Option[CallContext]): OBPReturnType[List[EndpointMappingT]] = Future{ + validateBankId(bankId, callContext) + (getEndpointMappingsCached(bankId), callContext) + } /** * Invalidate the Redis-backed resource-doc caches whose contents include diff --git a/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnectorInternal.scala b/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnectorInternal.scala index 9d05ca346c..0467216cd9 100644 --- a/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnectorInternal.scala +++ b/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnectorInternal.scala @@ -32,7 +32,7 @@ import com.openbankproject.commons.model._ import com.openbankproject.commons.model.enums.ChallengeType.OBP_TRANSACTION_REQUEST_CHALLENGE import com.openbankproject.commons.model.enums.TransactionRequestTypes._ import com.openbankproject.commons.model.enums.{TransactionRequestStatus, _} -import com.tesobe.CacheKeyFromArguments +import com.tesobe.{CacheKeyFromArguments, CacheKeyOmit} import net.liftweb.common._ import org.json4s.JsonAST.JValue import org.json4s.native.Serialization.write @@ -478,7 +478,14 @@ object LocalMappedConnectorInternal extends MdcLoggable { Full(cardList) } - def getCurrentFxRateCached(bankId: BankId, fromCurrencyCode: String, toCurrencyCode: String, callContext: Option[CallContext]): Box[FXRate] = { + // @CacheKeyOmit on callContext: the rate depends on the bank and the currency pair only, but + // CacheKeyFromArguments renders every un-annotated parameter into the key, and CallContext + // carries per-request state (startTime, correlationId, url, verb, ipAddress, user). Keying on + // it made the key unique per request: the cache could never hit, and every call wrote a fresh + // Redis entry that lived out code.fx.exchangeRate.cache.ttl.seconds. The generated connectors + // have always annotated their callContext (see ConnectorBuilderUtil); this hand-written site + // simply never did. + def getCurrentFxRateCached(bankId: BankId, fromCurrencyCode: String, toCurrencyCode: String, @CacheKeyOmit callContext: Option[CallContext]): Box[FXRate] = { /** * Please note that "var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)" * is just a temporary value field with UUID values in order to prevent any ambiguity. From 65561a3bfa43da83a3078c04df7f3bf2c8613aa9 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 26 Aug 2026 15:51:19 +0200 Subject: [PATCH 15/59] fix: scope the idempotency key by operation, not just consumer The scope key was derived from the consumer id alone, with no method, path, or resolved-operation component. Reusing one Idempotency-Key across two different endpoints (same or empty request body on both, e.g. two DELETEs) made the second call replay the first's cached response instead of executing: the caller was told an operation succeeded that never ran. This risk existed narrowly within v7.0.0 before the middleware was wired onto every version tree; wiring it onto all ~17 trees widened it to the entire API surface, and the middleware's own test only covered consumer-scoping, not endpoint-scoping. Fold the resolved operation id into the scope hash. operationId is set by ResourceDocMiddleware once it matches a ResourceDoc, and is stable across path-template placeholders and bridge-cascade path rewrites, unlike the raw request path. A tree that finds no ResourceDoc match falls back to method+path; that fallback only ever feeds a lock-then-release cycle a miss discards, so it does not need to be canonical, only present. Added a regression test reproducing the collision (two different endpoints, same consumer, same key, empty bodies on both sides) and updated the in-flight test's manually-planted lock key, whose comment already said "same derivation the middleware uses" -- it now is. --- .../util/http4s/IdempotencyMiddleware.scala | 23 ++++++++++++++---- .../http4s/IdempotencyMiddlewareTest.scala | Bin 14085 -> 15992 bytes 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/http4s/IdempotencyMiddleware.scala b/obp-api/src/main/scala/code/api/util/http4s/IdempotencyMiddleware.scala index 53e7bf6b37..55633d4041 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/IdempotencyMiddleware.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/IdempotencyMiddleware.scala @@ -31,9 +31,12 @@ import java.util.Base64 * - Concurrent replay while the original is still in flight: 409 Conflict. * - 5xx responses are NOT cached; clients can retry. * - * Scope: the key is namespaced by SHA-256 of the consumer id, or — when - * unauthenticated — the Authorization header. This prevents key reuse across - * consumers. + * Scope: the key is namespaced by SHA-256 of (consumer id, or — when + * unauthenticated — the Authorization header) AND the resolved operation id (or, absent a + * ResourceDoc match, the method+path). This prevents key reuse across consumers AND across + * endpoints: a client accidentally or deliberately reusing one Idempotency-Key against two + * different operations gets two independent dedup slots instead of the second operation being + * treated as a replay of the first and never executed. * * Validation: 8..255 printable-ASCII characters. Anything else → 400. * @@ -215,12 +218,22 @@ object IdempotencyMiddleware extends MdcLoggable { private def scopeFor(req: Request[IO]): String = { val ccOpt = req.attributes.lookup(Http4sRequestAttributes.callContextKey) - val raw = ccOpt + val consumerOrAuth = ccOpt .flatMap(_.consumer.map(_.consumerId.get).toOption) .filter(_.nonEmpty) .orElse(req.headers.get(AuthorizationHeader).map(_.head.value)) .getOrElse("anonymous") - sha256Hex(raw).take(16) + // operationId is the canonical identity of "which endpoint" -- set by ResourceDocMiddleware + // once it has matched a ResourceDoc, and stable across path-template placeholders and + // bridge-cascade path rewrites (v400->v310->...). A tree this request is NOT served by never + // gets a ResourceDoc match, so it falls back to method+path; that fallback only ever feeds a + // lock-then-release cycle that a miss discards (see runAndCache), so it does not need to + // identify anything durable -- it only needs to differ from a genuinely different endpoint's + // fallback often enough that misses don't manufacture cross-endpoint collisions of their own. + val endpoint = ccOpt + .flatMap(_.operationId) + .getOrElse(s"${req.method.name} ${req.uri.path.renderString}") + sha256Hex(s"$consumerOrAuth|$endpoint").take(16) } // ── Body hash ────────────────────────────────────────────────────────── diff --git a/obp-api/src/test/scala/code/api/util/http4s/IdempotencyMiddlewareTest.scala b/obp-api/src/test/scala/code/api/util/http4s/IdempotencyMiddlewareTest.scala index 80a135ae1138156bd94e249a0d642adc3161deb3..1a23aaac7932c408cf2a5c45a725fd1e1ae1baa3 100644 GIT binary patch delta 1519 zcmZ8hON$#v5GFnZTQSDYVG|Z3Dib*?VXfp{FR_We3(Ce|2aJR5bD%X{qiMFMd(w}U z#jY(3@zDn}kZb-x4uQQVe<8mmryNt=^H7KoQcq3SL-V_#wRzt?j2sj#XN^Y6aMRaU^Kr6X|-Ca-`3Us;V<8UmQ z%j4k`jTKr=V#U)b%u=JRjUlLbNr6kLR|P6JhR4p8>jOJ0ODd@-wM7eW_U=Iw1Drv! zEQQNqqRk^|5t!nD^nOKkV4Fi{A$|b_U};!cQr$w4iJl9=xx^(iI0nKND0ne@`e?(< zx}zrAJD6<2gtD$p;g%YxW?_)fbCG3e=#fk{-;~H(M?^S~=wcd+6pKU3q=Y$AA=a@I zRFG){jJi@vMj*0vmw`&lgt2Y{GX+?bZUuAA15o-W3asG724M;ykXeD0aY;cO;WA>e zKm)>k6P+G2n-7y@1c!%h{1B@(M=q$?T&Uv(u(|ei%_Un&%{Vxevd$B52M88e$&3q) zj{*oagq{P}XsZu*Kff9KyxS6zjAGtX>No|hZfxyTclzVrs5S@FpPvq@_4bu7={y3P zYcKh?!~ml%_d4Ht0?2*qW z!CCOA{^*Zsio#rtUPm<#&Ym=?DEoqR# zI2A%IV9wH$acoXg+LBpNL9BqUF!eOJFmH$sl#)H%a9WaF9VBo&!mcNser*z76XAjD z&6EbhBR+nMBt?n6u=z%a6QHi2HIX9X*y;?XOj!I7$4+Dct|Ned5-iLPU<9KLdsO^P+&Bp60jK1Zv)N%1?KKJ5k{ zEK)>4`z9_k1_g7Wh47!-qi#EF7RDwxiIN5&^f$<@!*ZA8Pwh#5c=qCDR0ko8v4a{y f(+vLP!;sFsko1dEFRQN(e(arh9v43!yuI`arMfS9G)gj36&y=TGV+Tus}f5x^YaukQWH~Bi*yvyGK-2! X6b#K2k~0#EiYND5$8G*@-N*z0!}S{( From 5b98bd8031859ad819de9f27235c53bf0b1571cf Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 26 Aug 2026 16:23:27 +0200 Subject: [PATCH 16/59] fix: don't report a DB failure during view lookup as 404 createTransactionRequest wrapped the whole view lookup (Views.views.vend .systemView(...).or(.customView(...))) inside tryons(..., 404, ...), which catches any Exception the block raises and reports it via the given failCode regardless of cause. A connection-pool exhaustion, a transient SQL error, or a Mapper bug during that lookup was therefore indistinguishable from a genuine "no such view" -- both produced the same 404, telling a client with retry logic to stop retrying a payment request that would in fact succeed once the backend recovered. Split the lookup out of the exception-swallowing block: only a lookup that completes successfully and returns an empty Box is a genuine client-side not-found and maps to 404; anything the lookup itself throws now propagates untouched, so it falls through to ErrorResponseConverter's catch-all (500) like any other unexpected server-side failure. Extracted into resolveCreateTransactionRequestView so the distinction is unit-testable without a live Mapper connection. --- .../scala/code/api/v4_0_0/Http4s400.scala | 37 +++++++-- .../v4_0_0/Http4s400ViewResolutionTest.scala | 76 +++++++++++++++++++ 2 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 obp-api/src/test/scala/code/api/v4_0_0/Http4s400ViewResolutionTest.scala diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala index eb6d64e4fa..48bbfa676f 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala @@ -3019,6 +3019,33 @@ object Http4s400 { // first synchronous read of `SS.user` captures the cc.user, then the Future chain // runs normally on any thread. + // Resolves the view createTransactionRequest needs, unit-testable without a live Mapper + // connection: `lookup` is production's real `Views.views.vend.systemView(...).or(...)` call + // in the route below, and a stub in the test. + // + // `lookup()` runs OUTSIDE tryons's blanket exception catch, not inside it. tryons/tryo catch + // any Exception the wrapped block raises and report it via the given failCode regardless of + // cause -- wrapping the DB call itself made a connection-pool exhaustion, a transient SQL + // error, or a Mapper bug indistinguishable from a genuine "no such view" and reported ALL of + // them as 404. `Future(lookup())` still catches an exception from `lookup()` (standard + // Future-block semantics), but as an ordinary failed Future carrying the ORIGINAL exception, + // untouched -- so it falls through to ErrorResponseConverter's catch-all (500), the same as + // any other unexpected server-side failure. Only a lookup that SUCCEEDS and returns an empty + // Box is a genuine client-side "not found", and only that case is explicitly mapped to 404 + // via tryons below (whose wrapped block cannot itself throw for any other reason -- it only + // ever raises the NoSuchElementException it constructs). + private[v4_0_0] def resolveCreateTransactionRequestView( + viewIdStr: String, + lookup: () => Box[View] + )(implicit cc: CallContext): Future[View] = + Future(lookup()).flatMap { + case Full(v) => Future.successful(v) + case _ => + NewStyle.function.tryons(s"$ViewNotFound Current view_id($viewIdStr)", 404, Some(cc)) { + throw new NoSuchElementException(s"view_id($viewIdStr)") + } + } + lazy val createTransactionRequest: HttpRoutes[IO] = HttpRoutes.of[IO] { // GRANT_VIEW_ID in the ResourceDoc URL → middleware skips view validation. // Lift's v4 endpoint does no view-access check upfront; it lets @@ -3061,16 +3088,10 @@ object Http4s400 { com.openbankproject.commons.util.JsonAliases.parse(bodyStr) } transactionRequestType = TransactionRequestType(transactionRequestTypeStr) - view <- NewStyle.function.tryons(s"$ViewNotFound Current view_id($viewIdStr)", 404, Some(cc)) { - // System views (owner, accountant, etc.) and custom views (e.g. VRP - // `_vrp-…` views) are stored separately. Try system first; fall back - // to the account-scoped custom view. SS.init only needs *some* View - // instance — the connector reads viewId from the parameter, not the - // View object — so a soft fallback is fine here. + view <- resolveCreateTransactionRequestView(viewIdStr, () => Views.views.vend.systemView(ViewId(viewIdStr)) .or(Views.views.vend.customView(ViewId(viewIdStr), BankIdAccountId(account.bankId, account.accountId))) - .openOrThrowException(s"$ViewNotFound Current view_id($viewIdStr)") - } + ) // SS.init populates Lift thread-globals (used by `SS.user` inside the // connector). The connector's first line `SS.user` resolves synchronously // inside this block, capturing the user; subsequent flatMap stages run on diff --git a/obp-api/src/test/scala/code/api/v4_0_0/Http4s400ViewResolutionTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/Http4s400ViewResolutionTest.scala new file mode 100644 index 0000000000..de94779a04 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v4_0_0/Http4s400ViewResolutionTest.scala @@ -0,0 +1,76 @@ +package code.api.v4_0_0 + +import code.api.util.CallContext +import com.openbankproject.commons.model.View +import net.liftweb.common.{Box, Empty} + +import scala.concurrent.Await +import scala.concurrent.duration._ + +/** + * `Http4s400.Implementations4_0_0.resolveCreateTransactionRequestView` decides between two + * failure shapes for `createTransactionRequest`'s view lookup: a genuinely missing view + * (client's fault, 404) and anything else the lookup throws (a connection-pool exhaustion, a + * transient SQL error, a Mapper bug -- none of them the client's fault, all of them a 500). + * + * `NewStyle.function.tryons` cannot tell these apart on its own: it catches any `Exception` the + * wrapped block raises and reports it via the caller-supplied failCode regardless of cause. If + * the whole DB lookup sits inside that block, an infra failure and a real not-found produce the + * identical JSON-encoded {"failCode":404,...} exception -- indistinguishable to + * ErrorResponseConverter, which resolves the 404 straight from that embedded field. A client + * whose retry logic reacts to 500 (retry) differently from 404 (stop) is told to stop when the + * backend is actually just broken. + * + * These tests call the production function directly with a stub `lookup`, so no live Mapper + * connection is needed to exercise the distinction -- but Implementations4_0_0's own static + * init registers the full v4.0.0 ResourceDoc set, which needs the app booted, hence + * V400ServerSetup rather than a bare unit-test base. + */ +class Http4s400ViewResolutionTest extends V400ServerSetup { + + private implicit val cc: CallContext = CallContext() + + private def resultOf[T](f: => T): Either[Throwable, T] = + try Right(f) catch { case t: Throwable => Left(t) } + + feature("createTransactionRequest's view lookup distinguishes not-found from infra failure") { + + scenario("a lookup that finds nothing fails with the JSON-encoded 404 envelope") { + val notFound: () => Box[View] = () => Empty + val outcome = resultOf(Await.result( + Http4s400.Implementations4_0_0.resolveCreateTransactionRequestView("nonexistent-view", notFound), + 5.seconds)) + + outcome match { + case Left(t) => + val msg = t.getMessage + withClue(s"expected a JSON envelope carrying failCode 404; got: $msg") { + msg should include("\"failCode\":404") + msg should include("View not found") + } + case Right(v) => + fail(s"expected the lookup to fail as not-found, but it returned $v") + } + } + + scenario("a lookup that throws for an infra reason propagates that exception, not a 404") { + val infraFailure = new RuntimeException("connection pool exhausted") + val broken: () => Box[View] = () => throw infraFailure + val outcome = resultOf(Await.result( + Http4s400.Implementations4_0_0.resolveCreateTransactionRequestView("some-view", broken), + 5.seconds)) + + outcome match { + case Left(t) => + val msg = t.getMessage + withClue(s"expected the original infra exception to propagate untouched so it resolves " + + s"to 500, not the 404 envelope reserved for a genuine not-found; got: $msg") { + msg should not include "\"failCode\":404" + msg should include("connection pool exhausted") + } + case Right(v) => + fail(s"expected the lookup's exception to propagate, but it returned $v") + } + } + } +} From 4f118099c3dc60f5730d0abf1f1580ccf0d3aaeb Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 26 Aug 2026 16:32:22 +0200 Subject: [PATCH 17/59] fix: don't report a broken security provider as an unsigned JWT createConsumerDynamicRegistration wrapped the whole JwtUtil.verifyJwt call in tryons(PostJsonIsNotSigned, 400, ...), which catches any Exception the block raises regardless of cause. verifyJwt goes through Nimbus JOSE (JWK.parseFromPEMEncodedObjects, SignedJWT.parse, RSASSAVerifier), and a missing signature algorithm in the JVM's registered security providers (a hardened/FIPS JRE, a stripped provider list, a provider-registration bug) surfaces there as a JOSEException wrapping NoSuchAlgorithmException -- caught by the same blanket tryons and reported to the caller as "your JSON is not signed" (400), even though nothing about their certificate or JWT is wrong and every other caller would fail identically until an operator fixes the JVM. Walk the exception's cause chain for NoSuchAlgorithmException/ NoSuchProviderException before deciding the status code: that shape propagates untouched (500, via ErrorResponseConverter's catch-all), while everything else -- a malformed PEM, an unparseable JWT, an actual signature mismatch -- still maps to 400 as before. Extracted into resolveJwtSignatureValid so the distinction is unit-testable without live PEM/JWT material. --- .../scala/code/api/v5_1_0/Http4s510.scala | 37 ++++++++- .../Http4s510JwtSignatureResolutionTest.scala | 75 +++++++++++++++++++ 2 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 obp-api/src/test/scala/code/api/v5_1_0/Http4s510JwtSignatureResolutionTest.scala diff --git a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala index c7f6ea11cb..ac899ea6a4 100644 --- a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala +++ b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala @@ -3041,6 +3041,39 @@ object Http4s510 { http4sPartialFunction = Some(createMyConsumer) ) + // Walks a Throwable's cause chain looking for a JVM/security-provider configuration problem + // (the requested algorithm or provider is unavailable) rather than anything about the + // caller-supplied certificate or JWT. `RSASSAVerifier`/`SignedJWT.verify` wrap + // NoSuchAlgorithmException in a JOSEException when the JVM's registered security providers + // don't have the requested signature algorithm (a hardened/FIPS JRE, a stripped provider + // list, a provider-registration bug) -- a server/environment fault that has nothing to do + // with whether this particular client's certificate is well-formed. + private[v5_1_0] def hasSecurityProviderCause(t: Throwable): Boolean = + Iterator.iterate(t)(_.getCause).takeWhile(_ != null).exists { + case _: java.security.NoSuchAlgorithmException => true + case _: java.security.NoSuchProviderException => true + case _ => false + } + + // `JwtUtil.verifyJwt` does not merely return false for a bad certificate -- it can THROW at + // several points (PEM parsing, JWT parsing, key extraction, signature verification), and a + // client-malformed certificate or JWT is exactly what most of those throws mean. But wrapping + // the whole call in tryons(..., 400, ...) also converted a JVM/security-provider failure (see + // hasSecurityProviderCause) into the same 400 -- telling a caller their input was bad when + // the truth is the server's environment cannot perform this verification for ANY caller. + // `verify` is a thunk rather than a direct call so this is testable without live PEM/JWT + // material: production passes `() => JwtUtil.verifyJwt(jwt, pem)`, the test a stub that + // throws a chosen exception. + private[v5_1_0] def resolveJwtSignatureValid( + verify: () => Boolean + )(implicit cc: code.api.util.CallContext): Future[Boolean] = + Future(verify()).recoverWith { + case t if hasSecurityProviderCause(t) => + Future.failed(t) + case t => + NewStyle.function.tryons(PostJsonIsNotSigned, 400, Some(cc)) { throw t } + } + val createConsumerDynamicRegistration: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ POST -> `prefixPath` / "dynamic-registration" / "consumers" => EndpointHelpers.executeFutureCreated(req) { @@ -3056,9 +3089,7 @@ object Http4s510 { // OBP-50000 / HTTP 500. A missing or malformed client certificate is a client error; // reporting it as a server fault tells a caller with retry logic to keep sending a // request that cannot ever succeed. - signatureValid <- NewStyle.function.tryons(PostJsonIsNotSigned, 400, Some(cc)) { - JwtUtil.verifyJwt(postedJwt.jwt, pem.getOrElse("")) - } + signatureValid <- resolveJwtSignatureValid(() => JwtUtil.verifyJwt(postedJwt.jwt, pem.getOrElse(""))) _ <- Helper.booleanToFuture(PostJsonIsNotSigned, 400, Some(cc)) { signatureValid } postedJson <- NewStyle.function.tryons(InvalidJsonFormat, 400, Some(cc)) { com.openbankproject.commons.util.JsonAliases.parse(JwtUtil.getSignedPayloadAsJson(postedJwt.jwt).getOrElse("{}")).extract[ConsumerPostJsonV510] diff --git a/obp-api/src/test/scala/code/api/v5_1_0/Http4s510JwtSignatureResolutionTest.scala b/obp-api/src/test/scala/code/api/v5_1_0/Http4s510JwtSignatureResolutionTest.scala new file mode 100644 index 0000000000..824353aac4 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v5_1_0/Http4s510JwtSignatureResolutionTest.scala @@ -0,0 +1,75 @@ +package code.api.v5_1_0 + +import code.api.util.CallContext + +import scala.concurrent.Await +import scala.concurrent.duration._ + +/** + * `Http4s510.Implementations5_1_0.resolveJwtSignatureValid` decides what a thrown exception from + * `JwtUtil.verifyJwt` means: a malformed client certificate or JWT (client's fault, 400) versus a + * JVM/security-provider configuration problem -- the requested signature algorithm is not + * registered (a hardened/FIPS JRE, a stripped provider list, a provider-registration bug). Both + * currently surface as the same `JOSEException`/generic `Exception`, and wrapping the whole call + * in `tryons(..., 400, ...)` cannot tell them apart: a security-provider fault would be reported + * to the caller as "your JSON is not signed" when nothing about their JWT is wrong -- the server + * cannot perform this verification for ANY caller until an operator fixes the JVM. + * + * These tests call the production function directly with a stub `verify`, so no real PEM/JWT + * material or JOSE library internals are needed to exercise the distinction. + */ +class Http4s510JwtSignatureResolutionTest extends V510ServerSetup { + + private implicit val cc: CallContext = CallContext() + + private def resultOf[T](f: => T): Either[Throwable, T] = + try Right(f) catch { case t: Throwable => Left(t) } + + feature("createConsumerDynamicRegistration's JWT verification distinguishes a bad client " + + "JWT from a broken security provider") { + + scenario("a verify() that returns false is a normal signature mismatch, not an error") { + val outcome = Await.result( + Http4s510.Implementations5_1_0.resolveJwtSignatureValid(() => false), 5.seconds) + outcome shouldBe false + } + + scenario("a verify() that throws for a malformed client JWT fails with the 400 envelope") { + val badJwt = new IllegalArgumentException("Invalid JWT serialization") + val outcome = resultOf(Await.result( + Http4s510.Implementations5_1_0.resolveJwtSignatureValid(() => throw badJwt), 5.seconds)) + + outcome match { + case Left(t) => + val msg = t.getMessage + withClue(s"expected a JSON envelope carrying failCode 400; got: $msg") { + msg should include("\"failCode\":400") + } + case Right(v) => + fail(s"expected verify() to fail as a bad JWT, but it returned $v") + } + } + + scenario("a verify() that throws because the JVM lacks the signature algorithm propagates " + + "that exception, not a 400") { + val providerFailure = + new com.nimbusds.jose.JOSEException("no such algorithm", + new java.security.NoSuchAlgorithmException("SHA256withRSA Signature not available")) + val outcome = resultOf(Await.result( + Http4s510.Implementations5_1_0.resolveJwtSignatureValid(() => throw providerFailure), 5.seconds)) + + outcome match { + case Left(t) => + val msg = t.getMessage + withClue(s"expected the original security-provider exception to propagate untouched " + + s"so it resolves to 500, not the 400 envelope reserved for a malformed " + + s"client JWT/certificate; got: $msg") { + msg should not include "\"failCode\":400" + msg should include("no such algorithm") + } + case Right(v) => + fail(s"expected verify()'s exception to propagate, but it returned $v") + } + } + } +} From b999a261f0f3559bbb160c1baa1647a0ddd2c189 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 26 Aug 2026 16:44:00 +0200 Subject: [PATCH 18/59] fix: report a missing portal URL prop as 503, not 400 resetPasswordUrl reported a missing public_obp_portal_url/portal_external_url as 400 -- a bare Future.failed(new Exception(s"$IncompleteServerConfiguration ...")) whose message starts with "OBP-10056: ", which ErrorResponseConverter's OBP-prefix path promotes only to {401,403,408,429} and defaults everything else to 400. An admin resetting a user's password was told their request was bad when the actual problem is an operator who hasn't configured the portal URL yet. This is the identical condition Http4s700's createTestEmail already reports as 503 in this same codebase, with the reasoning that a 500 (or, worse, a 400) tells a caller with retry logic the fault is transient or their own, when neither resolves without an operator editing props. Route the same message through tryons with an explicit 503 instead of a bare exception, so it bypasses the 400 default entirely. Extracted into resolveResetPasswordPortalUrl so the distinction is unit-testable without touching Props. --- .../scala/code/api/v6_0_0/Http4s600.scala | 31 +++++++++-- .../Http4s600ResetPasswordPortalUrlTest.scala | 52 +++++++++++++++++++ 2 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 obp-api/src/test/scala/code/api/v6_0_0/Http4s600ResetPasswordPortalUrlTest.scala diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index b4559f7cf2..64a5b9fdb5 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -1022,6 +1022,32 @@ object Http4s600 { } + // Resolves the portal URL used to build the reset-password link. Unit-testable without + // touching Props: `portalUrlBox` is production's real `APIUtil.getPortalUrl` call in the + // route below, and a fixed Box in the test. + // + // 503, not 400. A missing public_obp_portal_url/portal_external_url is an operator's + // configuration mistake, not this caller's -- the exact condition Http4s700's createTestEmail + // reports as 503 ("the server is not broken -- it is not configured to do this, and [a wrong + // code] tells a caller with retry logic that the fault is transient"). A bare + // Future.failed(new Exception(s"$IncompleteServerConfiguration ...")) resolves to 400: the + // message starts with "OBP-10056: ", which ErrorResponseConverter's OBP-prefix path promotes + // only to {401,403,408,429} and defaults everything else to 400 -- so the admin resetting a + // password is told their request was bad. tryons with an explicit failCode bypasses that + // default entirely. + private[v6_0_0] def resolveResetPasswordPortalUrl( + portalUrlBox: net.liftweb.common.Box[String] + )(implicit cc: CallContext): Future[String] = + portalUrlBox match { + case Full(url) => Future.successful(url) + case _ => + NewStyle.function.tryons( + s"$IncompleteServerConfiguration public_obp_portal_url (or legacy portal_external_url) is not set", + 503, Some(cc)) { + throw new NoSuchElementException("public_obp_portal_url") + } + } + // Route: POST /obp/v6.0.0/management/user/reset-password-url (201) lazy val resetPasswordUrl: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ POST -> `prefixPath` / "management" / "user" / "reset-password-url" => @@ -1048,10 +1074,7 @@ object Http4s600 { case _ => throw new Exception("User not found, not validated, or email mismatch") } } - portalUrl <- APIUtil.getPortalUrl match { - case Full(url) => Future.successful(url) - case _ => Future.failed(new Exception(s"$IncompleteServerConfiguration public_obp_portal_url (or legacy portal_external_url) is not set")) - } + portalUrl <- resolveResetPasswordPortalUrl(APIUtil.getPortalUrl) resetLink <- Future { val user: AuthUser = authUser user.uniqueId.set(java.util.UUID.randomUUID().toString.replace("-", "")) diff --git a/obp-api/src/test/scala/code/api/v6_0_0/Http4s600ResetPasswordPortalUrlTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/Http4s600ResetPasswordPortalUrlTest.scala new file mode 100644 index 0000000000..edd6e63973 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v6_0_0/Http4s600ResetPasswordPortalUrlTest.scala @@ -0,0 +1,52 @@ +package code.api.v6_0_0 + +import code.api.util.CallContext +import net.liftweb.common.{Empty, Full} + +import scala.concurrent.Await +import scala.concurrent.duration._ + +/** + * `Http4s600.Implementations6_0_0.resolveResetPasswordPortalUrl` reports what happens when + * `public_obp_portal_url` (or the legacy `portal_external_url`) isn't configured. That is an + * operator's configuration mistake, not a caller's -- the same condition this PR's own + * createTestEmail fix (Http4s700.scala) deliberately reports as 503, not 500, with the reasoning + * "the server is not broken -- it is not configured to do this, and [a wrong code] tells a + * caller with retry logic that the fault is transient." A missing portal URL should get the same + * treatment here: 503, not 400 -- an admin resetting a user's password should not be told their + * request was bad when the truth is nobody has configured the portal URL yet. + */ +class Http4s600ResetPasswordPortalUrlTest extends V600ServerSetup { + + private implicit val cc: CallContext = CallContext() + + private def resultOf[T](f: => T): Either[Throwable, T] = + try Right(f) catch { case t: Throwable => Left(t) } + + feature("resetPasswordUrl reports a missing portal URL as a server misconfiguration, not a client error") { + + scenario("a configured portal URL is used as-is") { + val url = Await.result( + Http4s600.Implementations6_0_0.resolveResetPasswordPortalUrl(Full("https://portal.example.com")), + 5.seconds) + url shouldBe "https://portal.example.com" + } + + scenario("an unconfigured portal URL fails with 503, not 400") { + val outcome = resultOf(Await.result( + Http4s600.Implementations6_0_0.resolveResetPasswordPortalUrl(Empty), 5.seconds)) + + outcome match { + case Left(t) => + val msg = t.getMessage + withClue(s"expected a JSON envelope carrying failCode 503 (an operator configuration " + + s"problem, not the caller's fault); got: $msg") { + msg should include("\"failCode\":503") + msg should not include "\"failCode\":400" + } + case Right(v) => + fail(s"expected the missing portal URL to fail, but it returned $v") + } + } + } +} From 277bddc15037256b5a5bfefd643ba7601e99721f Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 26 Aug 2026 17:04:42 +0200 Subject: [PATCH 19/59] test: make the failure/auth sweep drift check able to fail SweepCoverageTest's "the failure sweep covers the same set the auth sweep does" scenario computed authScope and failureScope from the exact same literal expression, typed out twice (catalog.filter(EndpointCatalog.skipReason(_).isEmpty).map(_.operationId).toSet). Two independent copies of one expression are equal by construction: the scenario could never fail, even after a real future divergence where one sweep grows a filter of its own and the endpoints that fall between the two are covered by neither -- silently, forever, because the guard was comparing two copies of itself rather than each sweep's actual coverage. Expose each sweep's own definition of what it covers as AuthSweepTest.scope / FailureSweepTest.scope, and have both the sweep's own byVersion grouping and SweepCoverageTest's comparison read that single definition. A change to either sweep's filtering is now automatically reflected on both sides of the comparison instead of needing to be kept in sync by hand. SweepCoverageDriftCheckTest scans SweepCoverageTest.scala's source rather than asserting at runtime: a value-equality check on the current catalog cannot distinguish "read from the real source" from "coincidentally equal duplicate" -- both produce the identical Set today, and the difference only matters for whether a FUTURE divergence gets caught, which no runtime assertion against today's catalog can exercise. --- .../scala/code/api/sweep/AuthSweepTest.scala | 15 +++- .../code/api/sweep/FailureSweepTest.scala | 14 +++- .../sweep/SweepCoverageDriftCheckTest.scala | 72 +++++++++++++++++++ .../code/api/sweep/SweepCoverageTest.scala | 16 +++-- 4 files changed, 106 insertions(+), 11 deletions(-) create mode 100644 obp-api/src/test/scala/code/api/sweep/SweepCoverageDriftCheckTest.scala diff --git a/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala b/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala index 966c6a821c..8be45be09e 100644 --- a/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala +++ b/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala @@ -47,6 +47,17 @@ import com.openbankproject.commons.util.JsonAliases.parse * scenario-per-endpoint layout would have given is preserved in that list — each line names the * operationId, the verb, the URL, the expectation and what actually came back. */ +object AuthSweepTest { + + /** + * The single definition of what this sweep covers. Exposed so SweepCoverageTest's drift check + * reads this directly instead of re-deriving its own copy of the same filter -- two copies of + * one expression are equal by construction and can never catch this sweep's own filtering + * changing independently of FailureSweepTest's. + */ + def scope: List[ResourceDoc] = EndpointCatalog.all.filter(EndpointCatalog.skipReason(_).isEmpty) +} + class AuthSweepTest extends ServerSetupWithTestData with DefaultUsers { object AuthSweep extends Tag("AuthSweep") @@ -208,9 +219,7 @@ class AuthSweepTest extends ServerSetupWithTestData with DefaultUsers { // ── the sweep, one scenario per version ───────────────────────────────────── private lazy val byVersion: Map[String, List[ResourceDoc]] = - EndpointCatalog.all - .filter(EndpointCatalog.skipReason(_).isEmpty) - .groupBy(_.implementedInApiVersion.toString) + AuthSweepTest.scope.groupBy(_.implementedInApiVersion.toString) feature("Every reachable endpoint enforces the authentication its ResourceDoc declares") { diff --git a/obp-api/src/test/scala/code/api/sweep/FailureSweepTest.scala b/obp-api/src/test/scala/code/api/sweep/FailureSweepTest.scala index 38c15abd75..27e7b5efe4 100644 --- a/obp-api/src/test/scala/code/api/sweep/FailureSweepTest.scala +++ b/obp-api/src/test/scala/code/api/sweep/FailureSweepTest.scala @@ -55,6 +55,17 @@ import com.openbankproject.commons.util.JsonAliases.parse * owns it; asserting it twice, from a place with no baseline to compare against, would produce * failures every time a message was reworded. */ +object FailureSweepTest { + + /** + * The single definition of what this sweep covers. Exposed so SweepCoverageTest's drift check + * reads this directly instead of re-deriving its own copy of the same filter -- two copies of + * one expression are equal by construction and can never catch this sweep's own filtering + * changing independently of AuthSweepTest's. + */ + def scope: List[ResourceDoc] = EndpointCatalog.all.filter(EndpointCatalog.skipReason(_).isEmpty) +} + class FailureSweepTest extends ServerSetupWithTestData with DefaultUsers { object FailureSweep extends Tag("FailureSweep") @@ -144,8 +155,7 @@ class FailureSweepTest extends ServerSetupWithTestData with DefaultUsers { "as correct") ) - private lazy val inScope: List[ResourceDoc] = - EndpointCatalog.all.filter(EndpointCatalog.skipReason(_).isEmpty) + private lazy val inScope: List[ResourceDoc] = FailureSweepTest.scope private def check(doc: ResourceDoc, headers: Map[String, String], ents: Map[String, String]): Option[String] = { diff --git a/obp-api/src/test/scala/code/api/sweep/SweepCoverageDriftCheckTest.scala b/obp-api/src/test/scala/code/api/sweep/SweepCoverageDriftCheckTest.scala new file mode 100644 index 0000000000..8d0c37a4b1 --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/SweepCoverageDriftCheckTest.scala @@ -0,0 +1,72 @@ +package code.api.sweep + +import org.scalatest.{FlatSpec, Matchers} + +import java.io.File +import scala.io.Source + +/** + * Guards SweepCoverageTest's "the failure sweep covers the same set the auth sweep does" + * scenario against being vacuous. + * + * That scenario computes `authScope` and `failureScope` and asserts their symmetric difference + * is empty -- but if both are built by literally re-typing the same filter expression twice + * (`catalog.filter(EndpointCatalog.skipReason(_).isEmpty).map(_.operationId).toSet`, written out + * in SweepCoverageTest.scala rather than read from AuthSweepTest/FailureSweepTest themselves), + * the two values are equal BY CONSTRUCTION regardless of what those two sweeps actually iterate + * over. The assertion can then never fail, even after a real future divergence -- one sweep + * gains a filter of its own, and the endpoints that fall between them are covered by neither, + * silently, forever, because the "guard" was checking two copies of itself. + * + * The fix is for SweepCoverageTest to read AuthSweepTest.scope and FailureSweepTest.scope -- + * each sweep's own single definition of what it covers -- instead of re-deriving a copy. This + * test is a source scan rather than a runtime assertion because a value-equality check on the + * CURRENT catalog cannot distinguish "computed from the real source" from "coincidentally equal + * duplicate" -- both produce the identical Set today; the difference only matters for whether a + * FUTURE divergence gets caught, which a static duplicate can never do regardless of what the + * catalog looks like when the test runs. + */ +class SweepCoverageDriftCheckTest extends FlatSpec with Matchers { + + private def sourceOf(basename: String): String = { + val candidates = List( + new File(s"src/test/scala/code/api/sweep/$basename"), + new File(s"obp-api/src/test/scala/code/api/sweep/$basename") + ) + val file = candidates.find(_.isFile).getOrElse( + fail(s"Cannot locate $basename under either candidate path - this guard must not pass by " + + s"failing to look. Tried: ${candidates.mkString(", ")}")) + val source = Source.fromFile(file, "UTF-8") + try source.mkString finally source.close() + } + + private lazy val sweepCoverageSource = sourceOf("SweepCoverageTest.scala") + + "the drift-check scenario" should "read AuthSweepTest's own scope, not a re-derived copy" in { + withClue("SweepCoverageTest must reference AuthSweepTest.scope so a future change to " + + "AuthSweepTest's own filtering is automatically reflected here instead of silently " + + "diverging from a hand-copied duplicate: ") { + sweepCoverageSource should include("AuthSweepTest.scope") + } + } + + it should "read FailureSweepTest's own scope, not a re-derived copy" in { + withClue("SweepCoverageTest must reference FailureSweepTest.scope for the same reason: ") { + sweepCoverageSource should include("FailureSweepTest.scope") + } + } + + it should "not compute authScope/failureScope by writing the skipReason filter out twice" in { + val duplicateFilterPattern = + """catalog\.filter\(EndpointCatalog\.skipReason\(_\)\.isEmpty\)\.map\(_\.operationId\)\.toSet""".r + val occurrences = duplicateFilterPattern.findAllIn(sweepCoverageSource).length + withClue(s"found $occurrences occurrence(s) of the raw filter expression written directly " + + s"in SweepCoverageTest.scala. Two occurrences means authScope and failureScope are " + + s"each an independent copy of the same literal, equal by construction regardless of " + + s"what AuthSweepTest/FailureSweepTest actually cover -- the exact vacuousness this " + + s"guard exists to catch. Expected zero: the scopes should come from " + + s"AuthSweepTest.scope / FailureSweepTest.scope instead. ") { + occurrences shouldBe 0 + } + } +} diff --git a/obp-api/src/test/scala/code/api/sweep/SweepCoverageTest.scala b/obp-api/src/test/scala/code/api/sweep/SweepCoverageTest.scala index 929321a8c8..7033ab47ff 100644 --- a/obp-api/src/test/scala/code/api/sweep/SweepCoverageTest.scala +++ b/obp-api/src/test/scala/code/api/sweep/SweepCoverageTest.scala @@ -92,12 +92,16 @@ class SweepCoverageTest extends ServerSetupWithTestData { } scenario("the failure sweep covers the same set the auth sweep does", SweepCoverage) { - // FailureSweepTest and AuthSweepTest read the same catalog through the same skipReason, - // so their scopes are equal by construction. Asserted anyway: if either ever grows a - // filter of its own, the two sweeps drift apart silently and the endpoints that fall - // between them are covered by neither. This is the cheapest possible guard against that. - val authScope = catalog.filter(EndpointCatalog.skipReason(_).isEmpty).map(_.operationId).toSet - val failureScope = catalog.filter(EndpointCatalog.skipReason(_).isEmpty).map(_.operationId).toSet + // Read each sweep's OWN scope rather than re-deriving a copy here: today both are + // `EndpointCatalog.all.filter(EndpointCatalog.skipReason(_).isEmpty)`, so two independently + // hand-typed copies of that expression would be equal by construction and this scenario + // would pass even after a real future divergence -- one sweep grows a filter of its own, + // the endpoints that fall between the two are covered by neither, and nothing here would + // notice. Reading AuthSweepTest.scope / FailureSweepTest.scope means there is exactly one + // definition of each sweep's coverage, so a change to either is automatically reflected + // on both sides of this comparison. + val authScope = AuthSweepTest.scope.map(_.operationId).toSet + val failureScope = FailureSweepTest.scope.map(_.operationId).toSet val onlyAuth = authScope -- failureScope val onlyFailure = failureScope -- authScope From 1129d0ae84833d3c45de6007e2dfbd8e1a679157 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 26 Aug 2026 22:04:29 +0200 Subject: [PATCH 20/59] fix: only run idempotency lock/response logic at the tier that matched operationId is set on the CallContext only when ResourceDocMiddleware's matcher finds a ResourceDoc for the current version tree; every other tree in the .orElse fallthrough chain sees it absent. Since this middleware is installed on every tree, a miss still acquired and released a lock keyed on a method+path fallback before falling through -- and two requests racing that same miss-tier lock (two genuinely concurrent copies of a request destined for a later tier, or two different endpoints whose fallback happened to collide) could have the loser answered a definite 409 by a tier that was never going to serve either of them, before the request ever reached its real destination. Gate the lock/response-key path on operationId being present. A miss-tier now passes straight through to the wrapped routes with no Redis round trip at all, removing both the false-conflict window and the redundant lookups on every tier a mutating request passes through before reaching the one that serves it. Key-format validation stays unconditional, since a malformed header is a client error regardless of which tier eventually resolves the path. --- .../util/http4s/IdempotencyMiddleware.scala | 67 +++++++++++++----- .../http4s/IdempotencyMiddlewareTest.scala | Bin 15992 -> 18328 bytes 2 files changed, 50 insertions(+), 17 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/http4s/IdempotencyMiddleware.scala b/obp-api/src/main/scala/code/api/util/http4s/IdempotencyMiddleware.scala index 55633d4041..d3d0429de2 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/IdempotencyMiddleware.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/IdempotencyMiddleware.scala @@ -32,13 +32,13 @@ import java.util.Base64 * - 5xx responses are NOT cached; clients can retry. * * Scope: the key is namespaced by SHA-256 of (consumer id, or — when - * unauthenticated — the Authorization header) AND the resolved operation id (or, absent a - * ResourceDoc match, the method+path). This prevents key reuse across consumers AND across - * endpoints: a client accidentally or deliberately reusing one Idempotency-Key against two - * different operations gets two independent dedup slots instead of the second operation being - * treated as a replay of the first and never executed. + * unauthenticated — the Authorization header) AND the resolved operation id. This prevents key + * reuse across consumers AND across endpoints: a client accidentally or deliberately reusing one + * Idempotency-Key against two different operations gets two independent dedup slots instead of + * the second operation being treated as a replay of the first and never executed. * - * Validation: 8..255 printable-ASCII characters. Anything else → 400. + * Validation: 8..255 printable-ASCII characters. Anything else → 400, regardless of whether this + * tier serves the path -- a malformed header is a client error no matter where it resolves. * * Storage: Redis via the existing JedisPool. Two keys per request: * - idem:lock:: → "1" (60s TTL, set with NX) @@ -52,17 +52,27 @@ import java.util.Base64 * INSIDE ResourceDocMiddleware, on every version's route tree. Both halves matter. * * Inside, because the scope key and the request-body hash both come from the CallContext, and - * ResourceDocMiddleware is what populates it. Mounted outside, `bodyFromCallContextOrEmpty` - * returns "" for every request, so two DIFFERENT payloads sent under one key hash identically -- - * the conflict check below silently stops working and the second caller receives the first - * caller's response. On a payment endpoint that is the difference between "your retry was - * deduplicated" and "your second, different payment returned someone else's receipt". + * ResourceDocMiddleware is what populates it. Mounted outside, there is no CallContext at all -- + * no operationId and no httpBody -- so `matchedThisTier` is false for every request and this + * middleware does nothing at all rather than mishandling anything: no lock, no cached response, + * no conflict detection. That is a silent loss of protection, not a silent corruption, but it is + * still a config-only guard against a real risk: on a payment endpoint, dedup quietly not running + * is the difference between "your retry was deduplicated" and "your retry submitted the payment + * again". * * On every tree, because Http4sApp composes the versions with `.orElse` and a tree signals "not * mine" by returning OptionT.none. This middleware therefore has to pass a miss through * unchanged; an earlier version answered 404 in that case, which terminated the chain -- measured * on POST /obp/v3.1.0/management/method_routings, which answered 201 without an Idempotency-Key - * and 404 with one. IdempotencyMiddlewareTest pins both properties. + * and 404 with one. A later version preserved the miss but still spent a full lock-acquire/release + * cycle doing it, using a method+path fallback scope for any tier that had not matched -- which + * meant two requests racing that SAME miss-tier's fallback lock (two genuinely different + * endpoints whose method+path fallback happened to differ less often than intended, or two + * concurrent copies of the request destined for a LATER tier) could have the loser answered a + * definite 409 by a tier that was never going to serve either of them, before the request ever + * reached the tier that would have handled it correctly. `matchedThisTier` closes that: only the + * one tier whose ResourceDocMatcher actually matched (the only tier where operationId is ever + * set) does any lock/response-key work at all. IdempotencyMiddlewareTest pins all of these. */ object IdempotencyMiddleware extends MdcLoggable { @@ -100,6 +110,21 @@ object IdempotencyMiddleware extends MdcLoggable { // A malformed header is a client error whatever the path resolves to, so this // one deliberately does NOT fall through. OptionT.liftF(invalidKeyResponse(key)) + } else if (!matchedThisTier(req)) { + // operationId is set on the CallContext ONLY by ResourceDocMiddleware.attachToCallContext, + // and only for the one version tree whose ResourceDocMatcher actually matched this + // request (see ResourceDocMiddleware.apply: the `case None` branch never attaches one). + // Every other tree in the `.orElse` chain is about to answer a miss regardless of what + // this middleware does, so doing lock/response-key work there is worse than wasted: two + // requests that legitimately belong to two DIFFERENT trees, or two genuinely concurrent + // copies of the same request, can race the SAME miss-tier's method+path-fallback lock + // and the loser gets a definite 409 here -- terminating the `.orElse` chain on behalf of + // a tree that was never going to serve either of them, before the request ever reaches + // the tree that would have handled it correctly. Skipping straight through at a + // miss-tier removes both the wasted Redis round trips and this false-conflict window; + // only the matching tier ever computes a scope key that means anything durable, so only + // it needs the protection. + routes.run(req) } else { val scope = scopeFor(req) val bodyHash = sha256Hex(bodyFromCallContextOrEmpty(req)) @@ -214,6 +239,15 @@ object IdempotencyMiddleware extends MdcLoggable { key.length <= MaxKeyLength && key.forall(c => c >= 0x21 && c <= 0x7E) + // True only for the one version tree whose ResourceDocMatcher matched this request -- + // ResourceDocMiddleware.attachToCallContext is the sole place operationId is ever set, and it + // runs only on a match (see ResourceDocMiddleware.apply's `case Some(resourceDoc)` branch; the + // `case None` branch attaches a CallContext with no operationId). Every other tree in the + // `.orElse` chain sees operationId absent here and skips idempotency handling entirely, because + // it is about to answer a miss regardless of what this middleware does. + private def matchedThisTier(req: Request[IO]): Boolean = + req.attributes.lookup(Http4sRequestAttributes.callContextKey).exists(_.operationId.isDefined) + // ── Scope ────────────────────────────────────────────────────────────── private def scopeFor(req: Request[IO]): String = { @@ -225,11 +259,10 @@ object IdempotencyMiddleware extends MdcLoggable { .getOrElse("anonymous") // operationId is the canonical identity of "which endpoint" -- set by ResourceDocMiddleware // once it has matched a ResourceDoc, and stable across path-template placeholders and - // bridge-cascade path rewrites (v400->v310->...). A tree this request is NOT served by never - // gets a ResourceDoc match, so it falls back to method+path; that fallback only ever feeds a - // lock-then-release cycle that a miss discards (see runAndCache), so it does not need to - // identify anything durable -- it only needs to differ from a genuinely different endpoint's - // fallback often enough that misses don't manufacture cross-endpoint collisions of their own. + // bridge-cascade path rewrites (v400->v310->...). scopeFor only ever runs after + // matchedThisTier has confirmed operationId is present, so the method+path fallback below is + // purely defensive -- it should never actually be exercised in production, only under a + // future bug that calls scopeFor without that guard. val endpoint = ccOpt .flatMap(_.operationId) .getOrElse(s"${req.method.name} ${req.uri.path.renderString}") diff --git a/obp-api/src/test/scala/code/api/util/http4s/IdempotencyMiddlewareTest.scala b/obp-api/src/test/scala/code/api/util/http4s/IdempotencyMiddlewareTest.scala index 1a23aaac7932c408cf2a5c45a725fd1e1ae1baa3..ada017f9bde3cdadebbd6ba617e482f00d1891a6 100644 GIT binary patch delta 2439 zcmZuzO^X~w7$znPh8T%L5)z{?Z6rI%%x+9VlHDZ>lI+475@mNmLB!N_zq8#;ch#h- zdMAU*oIFYlebApEWqDC~ui|va+*Wyh+i1^^ zJURA`)WEYPMV3{5l)@MGFO-kdE)99Y3H2v5Qa&YTXlbM-Q&KviCCh^nzx$l3JtpPI zj}0Z6^O=r)y#bzOos3MTF{sxgpK@!&^TUzR`GgV!1i6Wa*x8MYcHF~w8XG$VFxGfF zbA<1=M_9nx(}H7e2JPU@os4Z~vHI=!r8C_Z7`jEq%=#b$R(r z^@L7MKRIBSJj;R$6WX#e*0Sw_^}typ3MUOzd1jz;2lAmqV5gXK8tBB9ZAPapN^6X5 zRWQMt(lgmoV`q?@67R8{kk1P459S~G^EL121E~3mX~<8FbUH_-z1X_Yl@aKAKCR{mb7NzDYyJKX96ZUVMkS6?1z^xp`z*#j6QMZ3aU2^kDKFw=F3Yh5 zcrC}A?Lkl+vl%hSEWA^%yx_2;c5J6h{St5mF_30?K1&v4aE>mTLCX~@I5Tz+lzR&+ zi-gU9YU}*kIk?zT!bv%zhh@fc7%hqfg{r@nPw#I;pB|d(+A4EPOE`Rrop;j!EO;PB z{<#ivt$n~9(WNl5PCkA{BF2n~q6A2|VP*y-SR1^+qg=rTfR7y$daFTC3o{|RlqkK+ zIDo7EIQ!0xPH?C^3xWqZ^bboL+XJ`AW#;cnC_5|Flk;2E>E5yG;|o{!kG%Kn&FbfC zZ%hTuJeoDh;)mym;ZG$wblq@Vr5y}O*Df!kXF^@&1A-6=wyq5*YUv9?8B?5RU=NZe zD2Ggkj*TeO^ktyQa&>F@HQH7MV+ESC$h1^m+_W$X0RgF=-CP>>IXB}K0$JKOvdRbn zfE}GjTI3*B#f@Dd8eVjZ^a0)5+}@t4jrsv3{X#oH3u(>SzSW;Ar>1qqqeP8HTBQ?0 zuMXfL85_`!Q-i+PYF!ah^q>LJE+q`jLBjccB{tLn=2SB9OA{QK>dPW9vQ6V=J9C##olBoWc#p32MSy1~Eb zG!)KOskbLOUg#XHj@>#ty$`AyNQ=g-SmerrZRnQzqQcw*=d^u1hWG1+ z*u|wAG8K+Uvxb1%uzf)x*xGysCO+JwR;b^2Rk~_87H2^L2Z}#@&&8=!vSwt6%K@ Jz52~d{{i(IJYxU= delta 986 zcmZWo&1w`u5N16H@sEf(EGnjUSXecitO|NFN)iJiAwhExJeb~@noN_Ku1R-K)&XH3 z!9!m>c=8E^_2|V1@EP(5UOiehn`|P^p%2|vUwuC{pFg$!eEfENEnJxIPM*)-zy8u^ z2t!J0$dt<>a2Zv30#Qr_LTaf1DOgu|zD~yD2j2%}5*Q+2QTiH2+H;2oGGXynJN)k4 z9PSC+CbypJ+O{vGd=D(D#GS`m$-FPugn;0tJjh)F{ssdWs`vmrK@_dP7F9exfl-DQ z!-I=A$9?c4f<&h&-xv?2s&f)>5z?@Bs=!k$0T(4PnKak2hd4tt(D>kFr$aq0?J>x7kRVO#fwg%^?*%Usp`8o^RCPy%Sai0ONd3@Dvh$5cfeOX60U zmrz_449ftI*(!&J6`B~uKV@pLydO-;A|EpC{~=+}T@6<|{jjreIdporL%6drq2BS0 zkay-L@7BJwUIEz?X1o{$^3o+srSjQo)Sm_c51(+R$~xhO4xuvam6+qe;1MTRQdx5-j6bIC)(M5vkL>>TvpLw%NmHKRy=ebhLz6w#GFRq`^NjZSwncX`ZN4!cf+s6 KrAcq|u=N*$8cibr From 2a9b87d569f21b1d456711de48b21da598187cf0 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 26 Aug 2026 22:15:28 +0200 Subject: [PATCH 21/59] fix: recognize PAYMENT_SERVICE, PAYMENT_PRODUCT and SCA_METHOD as sweep placeholders isPlaceholder's ends-with-ID/_CODE/_NAME heuristic left these three ALL_CAPS segments untouched, treating them as literals. All three are enumerated values a live endpoint validates inline, not literals: Http4sBGv2PIS's payment-service branches guard on Set("payments","bulk-payments","periodic-payments").contains(paymentService), and the auth-context-updates/consent SCA branches guard on List(SMS, EMAIL[, IMPLICIT]).contains(scaMethod). Sent verbatim as "PAYMENT_SERVICE"/"SCA_METHOD", both guards fail and the sweep reports the resulting 404/400 as the endpoint's own defect -- reproducing today for SCA_METHOD via OBPv5.0.0-createUserAuthContextUpdateRequest. Add all three to isPlaceholder and give each a real accepted value in concretePath's defaultValue, the same way VIEW_ID resolves to "owner" so an endpoint's real logic runs instead of failing at an entity check before the assertion under test ever gets exercised. --- .../code/api/sweep/EndpointCatalog.scala | 34 ++++++-- .../code/api/sweep/EndpointCatalogTest.scala | 87 +++++++++++++++++++ 2 files changed, 113 insertions(+), 8 deletions(-) create mode 100644 obp-api/src/test/scala/code/api/sweep/EndpointCatalogTest.scala diff --git a/obp-api/src/test/scala/code/api/sweep/EndpointCatalog.scala b/obp-api/src/test/scala/code/api/sweep/EndpointCatalog.scala index 6f6eae87eb..c5bba1dd2a 100644 --- a/obp-api/src/test/scala/code/api/sweep/EndpointCatalog.scala +++ b/obp-api/src/test/scala/code/api/sweep/EndpointCatalog.scala @@ -97,18 +97,30 @@ object EndpointCatalog { // (CONSENTID, ACCOUNTID, BASKETID, DOMESTICPAYMENTID …), and leaving those literal sent // the string "CONSENTID" to the server as though it were an id. (seg.endsWith("ID") || seg.endsWith("_CODE") || seg.endsWith("_NAME") || - seg == "PROVIDER" || seg == "USERNAME" || seg == "USER_EMAIL") + seg == "PROVIDER" || seg == "USERNAME" || seg == "USER_EMAIL" || + // Named individually because none of the three ends in ID/_CODE/_NAME, yet each is a + // genuine enumerated-value placeholder a live endpoint validates inline, not a literal: + // Http4sBGv2PIS's payment-service branches guard on + // Set("payments","bulk-payments","periodic-payments").contains(paymentService), and + // Http4s310/Http4s400's auth-context-updates and consent SCA branches guard on + // List(StrongCustomerAuthentication.SMS, EMAIL[, IMPLICIT]).contains(scaMethod). Left as + // the literal strings "PAYMENT_SERVICE"/"SCA_METHOD", both guards fail and the sweep + // reports the resulting 404/400 as the endpoint's own defect -- confirmed reproducing + // today for SCA_METHOD via OBPv5.0.0-createUserAuthContextUpdateRequest. + // PAYMENT_PRODUCT is not independently validated anywhere it appears, but is named here + // for the same reason PROVIDER/USERNAME/USER_EMAIL are: it identifies a Berlin Group + // payment product, not a literal segment, and treating it as one just because nothing + // currently checks its value the way PAYMENT_SERVICE and SCA_METHOD are checked would be + // an accident of the present call sites, not the actual contract of the segment. + seg == "PAYMENT_SERVICE" || seg == "PAYMENT_PRODUCT" || seg == "SCA_METHOD") // Checked against Http4sSupport's literalAllCapsSegments: not one of the sixteen ends in ID, // _CODE or _NAME, so the rule above separates the two sets cleanly. // - // Known limitation, stated rather than papered over: a handful of ALL_CAPS segments are - // neither — PAYMENT_SERVICE and PAYMENT_PRODUCT take enumerated Berlin Group values, - // SCA_METHOD likewise, and CARDANO / MOBILE_WALLET / ETH_SEND_TRANSACTION are literals this - // list does not carry. Those stay verbatim, which routes them to the endpoint that expects - // that literal (correct for the literals) or produces a 404 (visible, and safe: a sweep - // reporting a 404 it caused is a nuisance, one silently substituting over a real literal is - // a coverage hole). + // CARDANO, MOBILE_WALLET and ETH_SEND_TRANSACTION are the remaining literals this list does + // not carry, and they stay verbatim on purpose -- substituting over them would route to the + // wrong case entirely (or none), which is the coverage hole this whole heuristic exists to + // avoid on the literal side. /** * The concrete path to call. @@ -140,6 +152,12 @@ object EndpointCatalog { // two defects stacked, the endpoint's raw throw AND this placeholder never resolving. case "VIEW_ID" | "GRANT_VIEW_ID" => "owner" case "USER_EMAIL" => "sweep-no-such-user@example.com" + // Enumerated values a live endpoint validates inline (see isPlaceholder) -- a well-formed + // but nonexistent value here would still fail that inline check, same as an id that does not + // exist fails a lookup, so these get one of the endpoint's own accepted values instead. + case "PAYMENT_SERVICE" => "payments" + case "PAYMENT_PRODUCT" => "sepa-credit-transfers" + case "SCA_METHOD" => "SMS" case _ => "sweep-no-such-" + seg.toLowerCase.replace('_', '-') } } diff --git a/obp-api/src/test/scala/code/api/sweep/EndpointCatalogTest.scala b/obp-api/src/test/scala/code/api/sweep/EndpointCatalogTest.scala new file mode 100644 index 0000000000..aae0179f01 --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/EndpointCatalogTest.scala @@ -0,0 +1,87 @@ +package code.api.sweep + +import code.setup.ServerSetupWithTestData +import org.scalatest.Tag + +/** + * EndpointCatalog.isPlaceholder decides which ALL_CAPS URL segments the sweeps substitute with a + * concrete value before calling an endpoint. Getting this wrong in either direction breaks a + * sweep's verdict: substituting a real literal sends a value the endpoint does not recognise, and + * leaving a real placeholder unsubstituted does the same thing from the other side -- the sweep + * calls a well-formed request that was never going to work, and reports the resulting 4xx/5xx as + * the endpoint's own defect. + * + * This pins the second failure mode for the three segments EndpointCatalog's own comment named as + * a known, accepted gap: PAYMENT_SERVICE, PAYMENT_PRODUCT and SCA_METHOD are enumerated values a + * live endpoint validates inline, not literals -- Http4sBGv2PIS's payment-service branches guard + * on `Set("payments", "bulk-payments", "periodic-payments").contains(paymentService)`, and + * Http4s310's auth-context-updates branch guards on + * `List(StrongCustomerAuthentication.SMS, EMAIL).contains(scaMethod)`. Left as the literal + * strings "PAYMENT_SERVICE"/"SCA_METHOD", both guards fail and the sweep calls a URL that was + * never going to route anywhere -- exactly the class of self-inflicted failure EndpointCatalog's + * own docstring says the ID/_CODE/_NAME heuristic exists to avoid. + */ +class EndpointCatalogTest extends ServerSetupWithTestData { + + object EndpointCatalogPlaceholders extends Tag("EndpointCatalogPlaceholders") + + feature("EndpointCatalog substitutes every real path placeholder, not just the ones ending in ID/_CODE/_NAME") { + + scenario("PAYMENT_SERVICE is substituted with a value the endpoint's own guard would accept", + EndpointCatalogPlaceholders) { + // No ResourceDoc in the current EndpointCatalog carries a PAYMENT_SERVICE segment -- + // Berlin Group's route trees are not aggregated into Http4s700.allResourceDocs (only the + // OBP v1.2.1..v7.0.0 lineage is), so this exercises concretePath/isPlaceholder directly via + // .copy on a real doc rather than filtering the live catalog for one that is not there. + // That pins the behaviour EndpointCatalog must have the day Berlin Group docs do join the + // catalog, instead of waiting to notice the gap then. + val doc = EndpointCatalog.all.head.copy( + requestUrl = "/PAYMENT_SERVICE/PAYMENT_PRODUCT/PAYMENT_ID/status") + val path = EndpointCatalog.concretePath(doc) + withClue(s"($path) left PAYMENT_SERVICE unsubstituted -- Http4sBGv2PIS's payment-status " + + s"branch guards on Set(\"payments\",\"bulk-payments\",\"periodic-payments\")" + + s".contains(paymentService), so this literal fails it and the sweep would " + + s"misreport a working endpoint as broken: ") { + path should not include "PAYMENT_SERVICE" + } + } + + scenario("SCA_METHOD is substituted with a value the endpoint's own guard accepts", + EndpointCatalogPlaceholders) { + val docs = EndpointCatalog.all.filter(_.requestUrl.contains("SCA_METHOD")) + withClue("no ResourceDoc in the catalog carries a SCA_METHOD segment any more -- this " + + "test's premise no longer holds against the current catalog, update it: ") { + docs should not be empty + } + docs.foreach { doc => + val path = EndpointCatalog.concretePath(doc) + withClue(s"${doc.operationId} ($path) left SCA_METHOD unsubstituted -- the endpoint " + + s"only accepts SMS/EMAIL/IMPLICIT, so this literal fails validation and the " + + s"sweep misreports a working endpoint as broken: ") { + path should not include "SCA_METHOD" + } + } + } + + scenario("literal ALL_CAPS segments that are not placeholders stay untouched", + EndpointCatalogPlaceholders) { + // The fix must not turn every unrecognised ALL_CAPS segment into a placeholder -- only the + // three named above. CARDANO/MOBILE_WALLET/ETH_SEND_TRANSACTION are genuine literals this + // catalog must keep sending verbatim. + val literalSegments = List("CARDANO", "MOBILE_WALLET", "ETH_SEND_TRANSACTION") + val docs = EndpointCatalog.all.filter(doc => literalSegments.exists(doc.requestUrl.contains)) + withClue("no ResourceDoc in the catalog carries any of CARDANO/MOBILE_WALLET/" + + "ETH_SEND_TRANSACTION any more -- this test's premise no longer holds, update it: ") { + docs should not be empty + } + docs.foreach { doc => + val path = EndpointCatalog.concretePath(doc) + val literalInDoc = literalSegments.find(doc.requestUrl.contains).get + withClue(s"${doc.operationId} substituted over the literal $literalInDoc, which routes " + + s"nowhere -- these are not placeholders: ") { + path should include(literalInDoc) + } + } + } + } +} From 8e27cc8dba5550fc989e3a37fa816dad82b8c41d Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 26 Aug 2026 22:26:56 +0200 Subject: [PATCH 22/59] fix: close the write-then-invalidate race on the endpoint-mapping cache Now that CallContext is out of getEndpointMappingsCached's key, the memoized entry can actually be hit -- which means the single delete in invalidateEndpointMappingCache leaves a real window: a reader that fetched the pre-write value from the provider a moment earlier can still finish its own cache write after the delete completes, silently reintroducing the stale entry for the rest of endpointMapping.cache.ttl.seconds with nothing left to clear it before the next write. Schedule a second delete shortly after the first, delay configurable via endpointMapping.cache.invalidation.delay.ms (default 500ms). Any straggler write that lands in the gap gets cleared moments later instead of surviving the full TTL. --- .../main/scala/code/api/util/NewStyle.scala | 19 +++++- ...EndpointMappingCacheInvalidationTest.scala | 59 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 obp-api/src/test/scala/code/api/util/EndpointMappingCacheInvalidationTest.scala diff --git a/obp-api/src/main/scala/code/api/util/NewStyle.scala b/obp-api/src/main/scala/code/api/util/NewStyle.scala index 08ba3303ed..ba7209efd6 100644 --- a/obp-api/src/main/scala/code/api/util/NewStyle.scala +++ b/obp-api/src/main/scala/code/api/util/NewStyle.scala @@ -3372,11 +3372,28 @@ object NewStyle extends MdcLoggable{ * This became necessary with the cache key fix below. While callContext was part of the * key nothing could ever hit, so a stale entry was unreachable by construction; now that * the cache works, writes have to publish themselves. + * + * A single delete here is not quite enough: a reader that fetched the pre-write value from + * the provider a moment earlier can still complete its own cache write AFTER this delete + * finishes, silently reintroducing the stale entry for the rest of the TTL -- nothing else + * would clear it until the next write. Scheduling a second delete closes that window the + * conventional way: any straggler write that lands in the gap gets cleared shortly after, + * long before an operator or caller would reasonably treat it as current. */ - private def invalidateEndpointMappingCache(): Unit = { + private[util] def invalidateEndpointMappingCache(): Unit = { Redis.deleteKeysByPattern("*getEndpointMappings*") + code.actorsystem.ObpActorSystem.localActorSystem.scheduler.scheduleOnce( + endpointMappingCacheInvalidationDelay + )(Redis.deleteKeysByPattern("*getEndpointMappings*"))( + code.actorsystem.ObpActorSystem.localActorSystem.dispatcher + ) + () } + private[util] val endpointMappingCacheInvalidationDelay: scala.concurrent.duration.FiniteDuration = + scala.concurrent.duration.FiniteDuration( + APIUtil.getPropsAsIntValue("endpointMapping.cache.invalidation.delay.ms", 500), "ms") + def getEndpointMappingById(bankId: Option[String], endpointMappingId : String, callContext: Option[CallContext]): OBPReturnType[EndpointMappingT] = { validateBankId(bankId, callContext) diff --git a/obp-api/src/test/scala/code/api/util/EndpointMappingCacheInvalidationTest.scala b/obp-api/src/test/scala/code/api/util/EndpointMappingCacheInvalidationTest.scala new file mode 100644 index 0000000000..e801297e0e --- /dev/null +++ b/obp-api/src/test/scala/code/api/util/EndpointMappingCacheInvalidationTest.scala @@ -0,0 +1,59 @@ +package code.api.util + +import code.api.JedisMethod +import code.api.cache.Redis +import code.setup.RedisTestTarget +import org.scalatest.{FlatSpec, Matchers} + +/** + * `NewStyle.function.invalidateEndpointMappingCache` guards against the write-then-invalidate + * race that comes with the cache key fix in `getEndpointMappingsCached`: while CallContext was + * part of the memoize key nothing could ever hit, so a stale entry was unreachable by + * construction. Now that the cache genuinely hits, a reader that fetched the pre-write value a + * moment earlier can still complete its own cache write AFTER the immediate delete finishes, + * silently reintroducing the stale entry for the rest of `endpointMapping.cache.ttl.seconds` -- + * nothing else would clear it before the next write. + * + * Racing two real threads against real DB/Redis latency would make this test flaky by + * construction (the window it is trying to hit is exactly the thing that is nondeterministic). + * Planting a key that matches the invalidation glob directly stands in for the straggler write + * instead -- the same deterministic-simulation technique IdempotencyMiddlewareTest uses for its + * own concurrency scenarios -- and this asserts the mechanism that is supposed to catch it: a + * second, delayed delete. + */ +class EndpointMappingCacheInvalidationTest extends FlatSpec with Matchers { + + private def redis(): Unit = + RedisTestTarget.requireReachable(Redis.isRedisReady, "the endpoint-mapping cache invalidation race guard") + + "invalidateEndpointMappingCache" should "clear a straggler entry that lands after the immediate delete" in { + redis() + // Any key matching the same glob the real memoized entry would (*getEndpointMappings*) + // stands in for the straggler -- the exact scalacache-derived key shape is not what this + // guards, only that a second sweep eventually clears whatever landed in the gap. + val stragglerKey = "test_ns:code.api.util.NewStyle.function.getEndpointMappingsCached(Some(straggler))()" + Redis.use(JedisMethod.SET, stragglerKey, None, Some("[]")) + withClue("test setup failed to plant the straggler key: ") { + Redis.use(JedisMethod.GET, stragglerKey, None, None) shouldBe Some("[]") + } + + NewStyle.function.invalidateEndpointMappingCache() + + withClue("immediately after the call the straggler should already be gone once, but that " + + "alone does not prove there is a SECOND delete -- see below") { + Redis.use(JedisMethod.GET, stragglerKey, None, None) shouldBe None + } + + // Simulate the race: the straggler write lands in the gap between the immediate delete and + // the scheduled one. + Redis.use(JedisMethod.SET, stragglerKey, None, Some("[]")) + + Thread.sleep(NewStyle.function.endpointMappingCacheInvalidationDelay.toMillis + 300) + + withClue("the delayed second invalidation must still clear a straggler that landed after " + + "the first delete, or a concurrent read racing the write can leave a stale " + + "endpoint-mapping list cached for the rest of the TTL: ") { + Redis.use(JedisMethod.GET, stragglerKey, None, None) shouldBe None + } + } +} From 705e9e52d2a86ce7d6fd9e8f4ce289058a1d4556 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 26 Aug 2026 22:34:59 +0200 Subject: [PATCH 23/59] fix: correct the zero-test floor diagnostic to quote the threshold it checks The floor comparison reads "${SF_TOTAL:-0}" -lt 3200, but the message printed on failure still said "(< 2000 floor)" -- left over from before the threshold was raised. A run producing e.g. 2500 tests is correctly failed by the check, but the printed diagnostic reads as self-contradictory (2500 is not less than 2000) to whoever is reading the CI log to find the cause. Add a source-scan test pinning that the two numbers match, the same drift-guard shape SweepCoverageDriftCheckTest already uses for a Scala file, so a future threshold change can't silently leave the message behind again. --- .../util/RunTestsParallelScriptTest.scala | 61 +++++++++++++++++++ run_tests_parallel.sh | 2 +- 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 obp-api/src/test/scala/code/util/RunTestsParallelScriptTest.scala diff --git a/obp-api/src/test/scala/code/util/RunTestsParallelScriptTest.scala b/obp-api/src/test/scala/code/util/RunTestsParallelScriptTest.scala new file mode 100644 index 0000000000..cbfb155bba --- /dev/null +++ b/obp-api/src/test/scala/code/util/RunTestsParallelScriptTest.scala @@ -0,0 +1,61 @@ +package code.util + +import org.scalatest.{FlatSpec, Matchers} + +import java.io.File +import scala.io.Source + +/** + * Pins that run_tests_parallel.sh's zero-test-floor diagnostic message quotes the same number + * it actually compares against. + * + * The floor check reads: + * + * if [[ "${SF_TOTAL:-0}" -lt 3200 ]]; then + * echo " ✗ suspicious total: only ${SF_TOTAL:-0} tests ran (< 2000 floor) ..." + * + * The threshold was raised from 2000 to 3200 (see the script's own comment: "3200 is 90% of the + * 3571 measured on develop-obp") but the message text was not updated alongside it. A run that + * produces, say, 2500 tests is correctly failed by the `-lt 3200` check, but the printed + * diagnostic reads "only 2500 tests ran (< 2000 floor)" -- which is arithmetically + * self-contradictory (2500 is not less than 2000) to whoever is reading the CI log to work out + * why the build failed. + * + * A runtime test cannot exercise a bash script's own comparison, so this reads the script's + * source and asserts the number in the `-lt` comparison matches the number quoted in the message + * -- the same drift-guard shape SweepCoverageDriftCheckTest uses for a Scala file. + */ +class RunTestsParallelScriptTest extends FlatSpec with Matchers { + + private def scriptSource: String = { + val candidates = List( + new File("run_tests_parallel.sh"), + new File("../run_tests_parallel.sh") + ) + val file = candidates.find(_.isFile).getOrElse( + fail(s"Cannot locate run_tests_parallel.sh under either candidate path - this guard must " + + s"not pass by failing to look. Tried: ${candidates.mkString(", ")}")) + val source = Source.fromFile(file, "UTF-8") + try source.mkString finally source.close() + } + + "the zero-test floor diagnostic" should "quote the same threshold it actually compares against" in { + val src = scriptSource + + val comparisonThreshold = """\$\{SF_TOTAL:-0\}"\s*-lt\s*(\d+)""".r + .findFirstMatchIn(src).map(_.group(1)).getOrElse( + fail("could not find the zero-test floor comparison (\"${SF_TOTAL:-0}\" -lt N) in the " + + "script - this guard must not pass by failing to look")) + + val messageThreshold = """\(<\s*(\d+)\s+floor\)""".r + .findFirstMatchIn(src).map(_.group(1)).getOrElse( + fail("could not find the \"(< N floor)\" diagnostic text in the script - this guard " + + "must not pass by failing to look")) + + withClue(s"the script compares against $comparisonThreshold but tells the reader the floor " + + s"is $messageThreshold -- whichever one is stale, a CI failure reads as " + + s"self-contradictory until they match: ") { + messageThreshold shouldBe comparisonThreshold + } + } +} diff --git a/run_tests_parallel.sh b/run_tests_parallel.sh index 2ec56d5b72..b37ff9e29a 100755 --- a/run_tests_parallel.sh +++ b/run_tests_parallel.sh @@ -548,7 +548,7 @@ fi # both numbers when the suite grows: a floor that is only half the real count is barely a # floor at all. if [[ "${SF_TOTAL:-0}" -lt 3200 ]]; then - echo " ✗ suspicious total: only ${SF_TOTAL:-0} tests ran (< 2000 floor) — filter/discovery regression?" + echo " ✗ suspicious total: only ${SF_TOTAL:-0} tests ran (< 3200 floor) — filter/discovery regression?" OVERALL_RC=1 fi From 9a2d841aeca9a1cd6bcb5ab9376cb8c4610806ac Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 26 Aug 2026 22:52:50 +0200 Subject: [PATCH 24/59] test: factor the duplicated sweep-fixture logic into one shared trait FailureSweepTest.omniscientUser and SuccessSweepTest.entitledCaller were byte-for-byte identical bodies (grant every ApiRole to resourceUser1, build a DirectLogin header), and AuthSweepTest, FailureSweepTest and SuccessSweepTest each looked up LocalMappedConnector.getBanksLegacy(None) independently. None of it lived in EndpointCatalog, the module this package already treats as the one place shared sweep logic belongs. Add SweepFixtures with realBankId and omniscientCaller, mixed into all three test classes; AuthSweepTest's realEntities now calls the shared realBankId instead of repeating the lookup. A source-scan test pins each construction to exactly one occurrence across the package, the same drift-guard shape SweepCoverageDriftCheckTest already uses. --- .../scala/code/api/sweep/AuthSweepTest.scala | 19 +++-- .../code/api/sweep/FailureSweepTest.scala | 33 +-------- .../code/api/sweep/SuccessSweepTest.scala | 22 +----- .../scala/code/api/sweep/SweepFixtures.scala | 41 +++++++++++ .../sweep/SweepFixturesDuplicationTest.scala | 69 +++++++++++++++++++ 5 files changed, 124 insertions(+), 60 deletions(-) create mode 100644 obp-api/src/test/scala/code/api/sweep/SweepFixtures.scala create mode 100644 obp-api/src/test/scala/code/api/sweep/SweepFixturesDuplicationTest.scala diff --git a/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala b/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala index 8be45be09e..6d1a29c275 100644 --- a/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala +++ b/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala @@ -58,7 +58,7 @@ object AuthSweepTest { def scope: List[ResourceDoc] = EndpointCatalog.all.filter(EndpointCatalog.skipReason(_).isEmpty) } -class AuthSweepTest extends ServerSetupWithTestData with DefaultUsers { +class AuthSweepTest extends ServerSetupWithTestData with DefaultUsers with SweepFixtures { object AuthSweep extends Tag("AuthSweep") @@ -98,16 +98,13 @@ class AuthSweepTest extends ServerSetupWithTestData with DefaultUsers { * over HTTP — the sweep is in-process and a round trip per lookup would be the only slow part * of it. */ - private lazy val realEntities: Map[String, String] = { - val bank = code.bankconnectors.LocalMappedConnector.getBanksLegacy(None).map(_._1).getOrElse(Nil).headOption - bank match { - case Some(b) => - val accountId = code.model.dataAccess.MappedBankAccount - .find(net.liftweb.mapper.By(code.model.dataAccess.MappedBankAccount.bank, b.bankId.value)) - .map(_.accountId.value) - Map("BANK_ID" -> b.bankId.value) ++ accountId.map("ACCOUNT_ID" -> _).toList.toMap - case None => Map.empty - } + private lazy val realEntities: Map[String, String] = realBankId match { + case Some(bankIdValue) => + val accountId = code.model.dataAccess.MappedBankAccount + .find(net.liftweb.mapper.By(code.model.dataAccess.MappedBankAccount.bank, bankIdValue)) + .map(_.accountId.value) + Map("BANK_ID" -> bankIdValue) ++ accountId.map("ACCOUNT_ID" -> _).toList.toMap + case None => Map.empty } private def describe(doc: ResourceDoc): String = diff --git a/obp-api/src/test/scala/code/api/sweep/FailureSweepTest.scala b/obp-api/src/test/scala/code/api/sweep/FailureSweepTest.scala index 27e7b5efe4..b941e1082e 100644 --- a/obp-api/src/test/scala/code/api/sweep/FailureSweepTest.scala +++ b/obp-api/src/test/scala/code/api/sweep/FailureSweepTest.scala @@ -3,9 +3,8 @@ package code.api.sweep import cats.effect.IO import cats.effect.unsafe.IORuntime import code.api.util.APIUtil.ResourceDoc -import code.api.util.{ApiRole, CustomJsonFormats} +import code.api.util.CustomJsonFormats import code.api.util.http4s.Http4sApp -import code.entitlement.Entitlement import code.setup.{DefaultUsers, ServerSetupWithTestData} import fs2.Stream import org.http4s.{Header, Headers, Method, Request, Uri} @@ -66,39 +65,13 @@ object FailureSweepTest { def scope: List[ResourceDoc] = EndpointCatalog.all.filter(EndpointCatalog.skipReason(_).isEmpty) } -class FailureSweepTest extends ServerSetupWithTestData with DefaultUsers { +class FailureSweepTest extends ServerSetupWithTestData with DefaultUsers with SweepFixtures { object FailureSweep extends Tag("FailureSweep") implicit val runtime: IORuntime = IORuntime.global private lazy val app = Http4sApp.httpApp - /** - * A caller holding every role in the system. - * - * Granted directly through the Entitlement provider rather than over the API — the same thing - * 161 existing test files do — because the goal is to get PAST authorisation, not to test it. - * Without this the sweep would stop at 403 on the ~478 role-gated endpoints and never reach - * the code that might crash. - */ - private def omniscientUser: Map[String, String] = { - ApiRole.availableRoles.foreach { role => - // Bank-scoped roles need a bank; system-wide ones must be granted with an empty bankId. - // valueOf throws on a name it does not recognise, and availableRoles includes dynamic - // roles whose backing entity may not exist in this database -- a grant that cannot be - // made is not a reason to abandon the other several hundred. - try { - val bankId = if (ApiRole.valueOf(role).requiresBankId) realBankId.getOrElse("") else "" - Entitlement.entitlement.vend.addEntitlement(bankId, resourceUser1.userId, role) - } catch { case _: Exception => () } - } - Map("DirectLogin" -> s"token=${token1.value}") - } - - private def realBankId: Option[String] = - code.bankconnectors.LocalMappedConnector.getBanksLegacy(None) - .map(_._1).getOrElse(Nil).headOption.map(_.bankId.value) - private def entities: Map[String, String] = realBankId.map("BANK_ID" -> _).toList.toMap @@ -183,7 +156,7 @@ class FailureSweepTest extends ServerSetupWithTestData with DefaultUsers { // table, so a lazy val granted during the first scenario leaves every later one // calling as an unentitled user -- which stops at 403 and never reaches the code // that might crash. That is how the first run reported only two 5xx. - val headers = omniscientUser + val headers = omniscientCaller val ents = entities val docs = byVersion(version) diff --git a/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala b/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala index 521d5b2be9..66502a2349 100644 --- a/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala +++ b/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala @@ -3,9 +3,8 @@ package code.api.sweep import cats.effect.IO import cats.effect.unsafe.IORuntime import code.api.util.APIUtil.ResourceDoc -import code.api.util.{ApiRole, CustomJsonFormats} +import code.api.util.CustomJsonFormats import code.api.util.http4s.Http4sApp -import code.entitlement.Entitlement import code.setup.{DefaultUsers, ServerSetupWithTestData} import fs2.Stream import org.http4s.{Header, Headers, Method, Request, Uri} @@ -57,7 +56,7 @@ import com.openbankproject.commons.util.JsonAliases.parse * `expectedNon2xx` with the reason, and asserted to STILL not be 5xx. An endpoint that stops * answering is a finding; an endpoint that was never going to answer here is a documented skip. */ -class SuccessSweepTest extends ServerSetupWithTestData with DefaultUsers { +class SuccessSweepTest extends ServerSetupWithTestData with DefaultUsers with SweepFixtures { object SuccessSweep extends Tag("SuccessSweep") @@ -97,21 +96,6 @@ class SuccessSweepTest extends ServerSetupWithTestData with DefaultUsers { "OBPv7.0.0-getRoutingScheme" -> "404 OBP-30514: no routing scheme named SCHEME" ) - private def realBankId: Option[String] = - code.bankconnectors.LocalMappedConnector.getBanksLegacy(None) - .map(_._1).getOrElse(Nil).headOption.map(_.bankId.value) - - /** A caller holding every role — the same construction FailureSweepTest uses. */ - private def entitledCaller: Map[String, String] = { - ApiRole.availableRoles.foreach { role => - try { - val bankId = if (ApiRole.valueOf(role).requiresBankId) realBankId.getOrElse("") else "" - Entitlement.entitlement.vend.addEntitlement(bankId, resourceUser1.userId, role) - } catch { case _: Exception => () } - } - Map("DirectLogin" -> s"token=${token1.value}") - } - private def get(path: String, headers: Map[String, String]): (Int, JValue) = { val req = Request[IO]( method = Method.GET, @@ -174,7 +158,7 @@ class SuccessSweepTest extends ServerSetupWithTestData with DefaultUsers { setPropsValues("api_disabled_endpoints" -> "[]", "api_enabled_endpoints" -> "[]") // Once per scenario -- beforeEach wipes the entitlement table, so a class-level // lazy val would leave every scenario after the first calling without roles. - val headers = entitledCaller + val headers = omniscientCaller val docs = byVersion(version) When(s"each of the ${docs.size} $version GETs that need no path variable is called") diff --git a/obp-api/src/test/scala/code/api/sweep/SweepFixtures.scala b/obp-api/src/test/scala/code/api/sweep/SweepFixtures.scala new file mode 100644 index 0000000000..c029f0bf56 --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/SweepFixtures.scala @@ -0,0 +1,41 @@ +package code.api.sweep + +import code.api.util.ApiRole +import code.entitlement.Entitlement +import code.setup.DefaultUsers + +/** + * Shared setup for sweeps that call the API with a fully-entitled caller. + * + * FailureSweepTest and SuccessSweepTest each grew an identical "grant every role, then build a + * DirectLogin header" construction, and AuthSweepTest, FailureSweepTest and SuccessSweepTest each + * looked up the fixture bank independently. One shared definition here, called from all three, + * means a future change to either only has one site to update. + */ +trait SweepFixtures { self: DefaultUsers => + + /** The first sandbox bank the fixtures created, if any. */ + def realBankId: Option[String] = + code.bankconnectors.LocalMappedConnector.getBanksLegacy(None) + .map(_._1).getOrElse(Nil).headOption.map(_.bankId.value) + + /** + * A caller holding every role in the system. + * + * Granted directly through the Entitlement provider rather than over the API -- the same thing + * 161 existing test files do -- because the goal is to get PAST authorisation, not to test it. + */ + def omniscientCaller: Map[String, String] = { + ApiRole.availableRoles.foreach { role => + // Bank-scoped roles need a bank; system-wide ones must be granted with an empty bankId. + // valueOf throws on a name it does not recognise, and availableRoles includes dynamic + // roles whose backing entity may not exist in this database -- a grant that cannot be + // made is not a reason to abandon the other several hundred. + try { + val bankId = if (ApiRole.valueOf(role).requiresBankId) realBankId.getOrElse("") else "" + Entitlement.entitlement.vend.addEntitlement(bankId, resourceUser1.userId, role) + } catch { case _: Exception => () } + } + Map("DirectLogin" -> s"token=${token1.value}") + } +} diff --git a/obp-api/src/test/scala/code/api/sweep/SweepFixturesDuplicationTest.scala b/obp-api/src/test/scala/code/api/sweep/SweepFixturesDuplicationTest.scala new file mode 100644 index 0000000000..ae947a1ec2 --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/SweepFixturesDuplicationTest.scala @@ -0,0 +1,69 @@ +package code.api.sweep + +import org.scalatest.{FlatSpec, Matchers} + +import java.io.File +import scala.io.Source + +/** + * Guards against the "grant every role" and "find a real bank id" constructions drifting back + * into byte-for-byte duplicates across the sweep test files. + * + * FailureSweepTest.omniscientUser and SuccessSweepTest.entitledCaller started as identical + * bodies -- grant every ApiRole to resourceUser1, then build a DirectLogin header -- and + * AuthSweepTest, FailureSweepTest and SuccessSweepTest each looked up + * LocalMappedConnector.getBanksLegacy(None) independently. None of it lived in EndpointCatalog, + * the module this package already treats as the one place shared sweep logic belongs (see + * AuthSweepTest.scope / FailureSweepTest.scope and SweepCoverageDriftCheckTest, which exists for + * exactly this reason on a different pair of definitions). + * + * A source scan, not a runtime assertion, for the same reason SweepCoverageDriftCheckTest is one: + * a value-equality check on today's fixtures cannot distinguish "computed from one shared + * definition" from "two copies that happen to still agree" -- both look identical today, and the + * difference only matters for whether a FUTURE divergence gets caught. + */ +class SweepFixturesDuplicationTest extends FlatSpec with Matchers { + + private def sourceOf(basename: String): String = { + val candidates = List( + new File(s"src/test/scala/code/api/sweep/$basename"), + new File(s"obp-api/src/test/scala/code/api/sweep/$basename") + ) + val file = candidates.find(_.isFile).getOrElse( + fail(s"Cannot locate $basename under either candidate path - this guard must not pass by " + + s"failing to look. Tried: ${candidates.mkString(", ")}")) + val source = Source.fromFile(file, "UTF-8") + try source.mkString finally source.close() + } + + "the 'grant every role, then build a DirectLogin header' construction" should + "appear once, in a shared fixture, not once per sweep file" in { + val pattern = """ApiRole\.availableRoles\.foreach""".r + val occurrences = List("SweepFixtures.scala", "FailureSweepTest.scala", "SuccessSweepTest.scala") + .map(f => f -> pattern.findAllIn(sourceOf(f)).length) + val total = occurrences.map(_._2).sum + + withClue(s"occurrences per file: ${occurrences.mkString(", ")}. Two independent copies of " + + s"the same 'grant every role' construction means a future change to how the " + + s"omniscient test caller is built (a new role category, a different bank-selection " + + s"rule) has to be applied by hand in both files, with nothing enforcing they stay " + + s"in sync. Expected exactly one, in a fixture both files call. ") { + total shouldBe 1 + } + } + + "the LocalMappedConnector.getBanksLegacy(None) bank lookup" should + "appear once, in a shared fixture, not once per sweep file" in { + val pattern = """LocalMappedConnector\.getBanksLegacy\(None\)""".r + val occurrences = List("SweepFixtures.scala", "AuthSweepTest.scala", "FailureSweepTest.scala", "SuccessSweepTest.scala") + .map(f => f -> pattern.findAllIn(sourceOf(f)).length) + val total = occurrences.map(_._2).sum + + withClue(s"occurrences per file: ${occurrences.mkString(", ")}. Three independent copies of " + + s"the same bank lookup means a future change to how the fixture bank is found has " + + s"to be applied by hand in three places, with nothing enforcing they stay in sync. " + + s"Expected exactly one, in a fixture all three files call. ") { + total shouldBe 1 + } + } +} From 0f1984c6ce242faa46e21e6637929e21df72e99c Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 26 Aug 2026 23:32:05 +0200 Subject: [PATCH 25/59] refactor: extract duplicated literals in CacheSerializationNamespaceTest SonarCloud flagged "code.example.Provider.getAll(Some(bank))" (3x) and "obpser1-scala2.13" (4x) as duplicated string literals. Named as SampleCallerKey and CurrentNamespace; no behavior change. --- .../CacheSerializationNamespaceTest.scala | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/obp-api/src/test/scala/code/api/cache/CacheSerializationNamespaceTest.scala b/obp-api/src/test/scala/code/api/cache/CacheSerializationNamespaceTest.scala index cc39d211be..af32030f35 100644 --- a/obp-api/src/test/scala/code/api/cache/CacheSerializationNamespaceTest.scala +++ b/obp-api/src/test/scala/code/api/cache/CacheSerializationNamespaceTest.scala @@ -38,14 +38,20 @@ import scalacache.{CacheConfig, DefaultCacheKeyBuilder} */ class CacheSerializationNamespaceTest extends FlatSpec with Matchers { + /** A stand-in caller key; its value is arbitrary, only its stability across calls matters. */ + private val SampleCallerKey = "code.example.Provider.getAll(Some(bank))" + + /** This build's namespace, spelled out rather than derived, so a test asserting against it + * fails loudly if the derivation and the literal ever disagree. */ + private val CurrentNamespace = "obpser1-scala2.13" + private def keyFor(namespace: String, callerKey: String): String = CacheConfig(cacheKeyBuilder = DefaultCacheKeyBuilder(keyPrefix = Some(namespace))) .cacheKeyBuilder.toCacheKey(Seq(callerKey)) "the derived cache key" should "differ between two serialization namespaces" in { - val caller = "code.example.Provider.getAll(Some(bank))" - val a = keyFor("obpser1-scala2.12", caller) - val b = keyFor("obpser1-scala2.13", caller) + val a = keyFor("obpser1-scala2.12", SampleCallerKey) + val b = keyFor(CurrentNamespace, SampleCallerKey) withClue(s"2.12 key <$a> and 2.13 key <$b> are the same, so a 2.13 instance would read the " + s"bytes a 2.12 instance wrote -- which is the defect this exists to prevent. ") { @@ -59,13 +65,11 @@ class CacheSerializationNamespaceTest extends FlatSpec with Matchers { // The Scala version does not move for a dependency upgrade that changes the encoding -- // chill 0.9.3 to 0.9.5 on its own would not have. The counter is the escape hatch for that, // and it is only an escape hatch if it actually changes the key. - val caller = "code.example.Provider.getAll(Some(bank))" - keyFor("obpser1-scala2.13", caller) should not equal keyFor("obpser2-scala2.13", caller) + keyFor(CurrentNamespace, SampleCallerKey) should not equal keyFor("obpser2-scala2.13", SampleCallerKey) } it should "stay stable for one namespace, or nothing would ever be a cache hit" in { - val caller = "code.example.Provider.getAll(Some(bank))" - keyFor("obpser1-scala2.13", caller) shouldBe keyFor("obpser1-scala2.13", caller) + keyFor(CurrentNamespace, SampleCallerKey) shouldBe keyFor(CurrentNamespace, SampleCallerKey) } "the namespace this build uses" should "name the Scala binary version it was compiled against" in { From 5d1f10331e0732c17bb941a4101cc7c83f2a4310 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 26 Aug 2026 23:47:25 +0200 Subject: [PATCH 26/59] refactor: extract duplicated literals in IdempotencyMiddlewareTest SonarCloud flagged "Idempotency-Key" (4x), "Bearer c1" (15x) and "/not-served-by-this-tree" (3x) as duplicated string literals. Named as IdempotencyKeyHeaderName, Consumer1Auth and UnservedPath; the three uri"..." literal constructions became Uri.unsafeFromString(UnservedPath) so they could reference the constant. No behavior change. --- .../http4s/IdempotencyMiddlewareTest.scala | Bin 18328 -> 18739 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/obp-api/src/test/scala/code/api/util/http4s/IdempotencyMiddlewareTest.scala b/obp-api/src/test/scala/code/api/util/http4s/IdempotencyMiddlewareTest.scala index ada017f9bde3cdadebbd6ba617e482f00d1891a6..bdc09eeab780d031a9017f142b428c7ca0158312 100644 GIT binary patch delta 914 zcma)4&ubGw6edlJw9xtk!5O;aL!&|AMnUNMVxCdhib=O?`KpHr5D7F(<0vEw z=@bCW1H_R+L1K9e82_gO6jNkW!Zv4ltF==Zi9sq6u`WEr0k?@>=3cfc5hxF{1escZ z`5RU^6;a=MIRc2`bgX4h^QOEMe}H~JaK-=!m&3`XI@NCxA>5OKB!#KhqIqu)Erd4j?Fkk zPoF$n(R-KH{)cDHIQ$%U74)Clu8WwRD?2A#NT05MbP@Bt{(H=Y^q0l`0h)%D%3Yl|r9b#g z|7v2vZgc5dc}`Yq4*ga8ac3QB&KSYU;W38Y4eFy~y$M=p93RiKUr)dHpPPHPe@qmT Jom+if`U~Rx8C0FWtr&cO$-p8NK%dO;;npl)tq>yZ= zG+97SaEmbp`gsfolb8F?zFN_M9WlM`PmGtxSOLU7 From 7d9e2e74b93c53896d0d59a40596de892e495e03 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 26 Aug 2026 23:55:23 +0200 Subject: [PATCH 27/59] refactor: extract duplicated literal in RedisTestTargetTest SonarCloud flagged "a check" as duplicated 6 times. Named as CheckLabel; no behavior change. --- .../scala/code/setup/RedisTestTargetTest.scala | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/obp-api/src/test/scala/code/setup/RedisTestTargetTest.scala b/obp-api/src/test/scala/code/setup/RedisTestTargetTest.scala index 48a09cddac..3d7d374c8c 100644 --- a/obp-api/src/test/scala/code/setup/RedisTestTargetTest.scala +++ b/obp-api/src/test/scala/code/setup/RedisTestTargetTest.scala @@ -17,27 +17,30 @@ import org.scalatest.exceptions.{TestCanceledException, TestFailedException} */ class RedisTestTargetTest extends FlatSpec with Matchers { + /** Arbitrary label passed as `what`; only its identity across calls matters, not its text. */ + private val CheckLabel = "a check" + "requireReachable" should "return normally when Redis is reachable, whether or not it is required" in { noException should be thrownBy RedisTestTarget.requireReachable( - reachable = true, what = "a check", required = false) + reachable = true, what = CheckLabel, required = false) noException should be thrownBy RedisTestTarget.requireReachable( - reachable = true, what = "a check", required = true) + reachable = true, what = CheckLabel, required = true) } it should "cancel when Redis is absent and optional" in { val e = intercept[TestCanceledException] { - RedisTestTarget.requireReachable(reachable = false, what = "a check", required = false) + RedisTestTarget.requireReachable(reachable = false, what = CheckLabel, required = false) } e.getMessage should include("Redis not reachable") - e.getMessage should include("a check") + e.getMessage should include(CheckLabel) } it should "FAIL, not cancel, when Redis is absent and required" in { val e = intercept[TestFailedException] { - RedisTestTarget.requireReachable(reachable = false, what = "a check", required = true) + RedisTestTarget.requireReachable(reachable = false, what = CheckLabel, required = true) } e.getMessage should include("OBP_TEST_REDIS_REQUIRED=true") - e.getMessage should include("a check") + e.getMessage should include(CheckLabel) } "required" should "read OBP_TEST_REDIS_REQUIRED and default to false" in { From 6ad14de880bf10490989b5cf77450b3c931ea245 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 15:46:05 +0200 Subject: [PATCH 28/59] fix: include Berlin Group v1.3 alias docs in the global resource-doc registry The resource-docs dispatcher serves the BG v1.3 alias (active only when berlin_group_v1_3_alias_path is set) through its ScannedApis registration, but APIUtil.allStaticResourceDocs never included it. Its docs carry their own operation ids, re-derived from the alias version string, so alias operation ids failed the getAllResourceDocs membership check used by api-collection-endpoint creation and other operation-id lookups -- the same gap BGv2 had before it was added to this union. Reproduced against a running instance with the alias prop set (OBP-40048 on a valid alias operation id) and confirmed the fix resolves it. --- obp-api/src/main/scala/code/api/util/APIUtil.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index 9e45f49432..b3d685f30b 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -4892,6 +4892,12 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ // ++ code.api.MxOF.OBP_MXOF_1_0_0.allResourceDocs // ++ code.api.BahrainOBF.v1_0_0.ApiCollector.allResourceDocs ++ code.api.berlin.group.v1_3.OBP_BERLIN_GROUP_1_3.allResourceDocs + // The BG v1.3 alias (active only when berlin_group_v1_3_alias_path is set) is served by + // the resource-docs dispatcher via its ScannedApis registration, but its docs carry their + // own operation ids (re-derived from the alias version), so it hit the same + // getAllResourceDocs membership gap as BGv2 below. Empty when the prop is unset, so this + // is a no-op on instances without the alias configured. + ++ code.api.berlin.group.v1_3.OBP_BERLIN_GROUP_1_3_Alias.allResourceDocs // BGv2 was missing here even though /resource-docs/BGv2 serves it, so a BGv2 operation id // (e.g. BGv2-getAccountDetails) failed the getAllResourceDocs membership check that // api-collection-endpoints (and anything else resolving operation ids) relies on. From f1e9f4ea66f562bef5d518cbe247d9e8d889b469 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 15:48:54 +0200 Subject: [PATCH 29/59] test: cover Berlin Group v1.3 alias in resource-doc registry parity test The alias surface is gated by berlin_group_v1_3_alias_path, which is unset in the default test environment, so its operation-id list is legitimately empty there -- skip the non-empty assertion for it while still running the membership check against getAllResourceDocs. --- .../util/ResourceDocRegistryParityTest.scala | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala index 520d04a223..fa0e25247a 100644 --- a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala +++ b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala @@ -11,11 +11,12 @@ import org.scalatest.Tag * * These are two parallel registries (ResourceDocsAPIMethods dispatches per * standard/version; getAllResourceDocs aggregates them all), and they have drifted - * twice: Berlin Group v2 was served by the dispatcher but missing from the global - * registry (so BGv2-getAccountDetails could not be added to an API collection), - * and the global registry was based on the v6 aggregation, excluding v7-only - * operation ids. When you add a NEW API standard, register its docs in BOTH - * places — and add its surface to this list. + * three times: Berlin Group v2 was served by the dispatcher but missing from the + * global registry (so BGv2-getAccountDetails could not be added to an API + * collection), the global registry was based on the v6 aggregation, excluding + * v7-only operation ids, and the Berlin Group v1.3 alias (active only when + * berlin_group_v1_3_alias_path is set) was missing too. When you add a NEW API + * standard, register its docs in BOTH places — and add its surface to this list. */ class ResourceDocRegistryParityTest extends ServerSetup { @@ -24,19 +25,24 @@ class ResourceDocRegistryParityTest extends ServerSetup { private lazy val allOperationIds: Set[String] = APIUtil.getAllResourceDocs.map(_.operationId).toSet - private lazy val surfaces: List[(String, Seq[String])] = List( - ("OBP standard (v7 aggregation)", code.api.v7_0_0.Http4s700.allResourceDocs.map(_.operationId).toSeq), - ("Berlin Group v1.3", code.api.berlin.group.v1_3.Http4sBGv13.resourceDocs.map(_.operationId).toSeq), - ("Berlin Group v2", code.api.berlin.group.v2.Http4sBGv2.resourceDocs.map(_.operationId).toSeq), - ("UK Open Banking 2.0.0", code.api.UKOpenBanking.v2_0_0.OBP_UKOpenBanking_200.allResourceDocs.map(_.operationId).toSeq), - ("UK Open Banking 3.1.0", code.api.UKOpenBanking.v3_1_0.OBP_UKOpenBanking_310.allResourceDocs.map(_.operationId).toSeq), - ("UK Open Banking 4.0.1", code.api.UKOpenBanking.v4_0_1.OBP_UKOpenBanking_401.allResourceDocs.map(_.operationId).toSeq) + // optional=true surfaces are gated by a prop that is unset by default (test props + // included), so their operation-id list is legitimately empty in most environments -- + // the membership check still runs (trivially true when empty) but the non-empty + // assertion is skipped for them. + private lazy val surfaces: List[(String, Seq[String], Boolean)] = List( + ("OBP standard (v7 aggregation)", code.api.v7_0_0.Http4s700.allResourceDocs.map(_.operationId).toSeq, false), + ("Berlin Group v1.3", code.api.berlin.group.v1_3.Http4sBGv13.resourceDocs.map(_.operationId).toSeq, false), + ("Berlin Group v1.3 alias", code.api.berlin.group.v1_3.Http4sBGv13Alias.resourceDocs.map(_.operationId).toSeq, true), + ("Berlin Group v2", code.api.berlin.group.v2.Http4sBGv2.resourceDocs.map(_.operationId).toSeq, false), + ("UK Open Banking 2.0.0", code.api.UKOpenBanking.v2_0_0.OBP_UKOpenBanking_200.allResourceDocs.map(_.operationId).toSeq, false), + ("UK Open Banking 3.1.0", code.api.UKOpenBanking.v3_1_0.OBP_UKOpenBanking_310.allResourceDocs.map(_.operationId).toSeq, false), + ("UK Open Banking 4.0.1", code.api.UKOpenBanking.v4_0_1.OBP_UKOpenBanking_401.allResourceDocs.map(_.operationId).toSeq, false) ) feature("getAllResourceDocs contains every per-standard resource-doc surface") { - surfaces.foreach { case (label, operationIds) => + surfaces.foreach { case (label, operationIds, optional) => scenario(s"$label operation ids are all resolvable globally", RegistryParityTag) { - operationIds should not be empty + if (!optional) operationIds should not be empty val missing = operationIds.filterNot(allOperationIds.contains) withClue(s"$label operation ids missing from getAllResourceDocs: ${missing.take(10).mkString(", ")} ") { missing shouldBe empty From f1344c1c232e489d6d734c1e27da486ef2eadce2 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 15:50:36 +0200 Subject: [PATCH 30/59] refactor: remove dead activeResourceDocs identity match in resource-docs dispatcher Every one of its ~19 arms was `case X => resourceDocs`, unchanged -- a leftover from the pre-http4s Lift route-filter era that stopped doing any filtering once the corresponding version moved fully onto http4s. getResourceDocsList now feeds resourceDocs directly into activePlusLocalResourceDocs, with identical output. --- .../ResourceDocsAPIMethods.scala | 28 +------------------ 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala index b14f064629..8263d92d79 100644 --- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala +++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala @@ -355,35 +355,9 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth logger.debug(s"There are ${resourceDocs.length} resource docs available to $requestedApiVersion") - val activeResourceDocs = requestedApiVersion match { - case ApiVersion.v7_0_0 => resourceDocs - case ConstantsBG.`berlinGroupVersion1` => resourceDocs // fully on http4s — no Lift route filter - case ConstantsBG.`berlinGroupVersion2` => resourceDocs - case ApiVersion.v1_2_1 => resourceDocs - case ApiVersion.v6_0_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v5_1_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v5_0_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v4_0_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v3_1_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v3_0_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v2_2_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v2_1_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v2_0_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v1_4_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v1_3_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.`dynamic-entity` => resourceDocs // runtime CRUD now on Http4sDynamicEntity; routes are Nil, skip Lift-route filter - case ApiVersion.`dynamic-endpoint` => resourceDocs // dispatch now on Http4sDynamicEndpoint (proxy + native Piece C); routes carry only the stub, skip Lift-route filter - case ApiVersion.ukOpenBankingV20 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.ukOpenBankingV31 => resourceDocs // fully on http4s — no Lift route filter - case _ => resourceDocs - } - - logger.debug(s"There are ${activeResourceDocs.length} resource docs available to $requestedApiVersion") - - val activePlusLocalResourceDocs = ArrayBuffer[ResourceDoc]() - activePlusLocalResourceDocs ++= activeResourceDocs + activePlusLocalResourceDocs ++= resourceDocs requestedApiVersion match { // only `obp` standard show the `localResourceDocs` From 215ca051085d9d81c13988c364ffc555350ce8ae Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 16:43:30 +0200 Subject: [PATCH 31/59] feat: add Java language support to DynamicResourceDoc method_body can now be Scala (default, unchanged) or Java, mirroring the existing precedent already used by ConnectorMethod/DynamicMessageDoc. - add programming_lang field to JsonDynamicResourceDoc / DynamicResourceDoc.Lang - compile Java method_body via the JSR-223 java engine into a native Http4sEndpointIO, reusing the same javax.tools.JavaCompiler backend as createJavaFunction - validate the actual compiled Java class (not its Scala wrapper) against dynamic_code_compile_validate_dependencies before it is ever invoked -- the wrapper alone can't see into a compiled Function reference (typically a synthetic lambda class) the way it can for a Scala closure - convert Java-native return values (Map/List/String/number/boolean) to JValue directly, since Extraction.decompose only reflects Scala types and silently drops a raw java.util.Map's entries - reject unsupported programming_lang values with a 400 before attempting compilation - thread programming_lang through DynamicResourceDocsEndpointGroup, which recompiles every registered doc when serving it -- without this a stored Java doc would silently be recompiled as Scala and fail at request time --- .../endpoint/helper/DynamicEndpoints.scala | 21 ++- .../DynamicResourceDocsEndpointGroup.scala | 2 +- .../scala/code/api/util/DynamicUtil.scala | 132 +++++++++++++++ .../scala/code/api/v4_0_0/Http4s400.scala | 9 +- .../DynamicResourceDoc.scala | 6 +- .../DynamicResourceDocProvider.scala | 7 +- .../MappedDynamicResourceDocProvider.scala | 2 + .../src/test/resources/frozen_type_meta_data | Bin 168937 -> 168869 bytes .../DynamicUtilJavaHttp4sEndpointTest.scala | 109 +++++++++++++ .../v4_0_0/DynamicResourceDocJavaTest.scala | 151 ++++++++++++++++++ 10 files changed, 433 insertions(+), 6 deletions(-) create mode 100644 obp-api/src/test/scala/code/api/util/DynamicUtilJavaHttp4sEndpointTest.scala create mode 100644 obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala index beaad62688..5d3ef7c1cc 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala @@ -77,7 +77,7 @@ trait EndpointGroup { * @param successResponseBody successResponseBody from the post json body,it is JValue here. * @param methodBody it is url-encoded string for the api level code. */ -case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String) { +case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String, programmingLang: String = "Scala") { val decodedMethodBody = URLDecoder.decode(methodBody, "UTF-8") val requestBody: Product = exampleRequestBody match { //this case means, we accept the empty string "" from json post body, we need to map it to None. @@ -87,7 +87,24 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo } val successResponse: Product = toCaseObject(successResponseBody) - private val partialFunction: Http4sEndpointIO = { + private val partialFunction: Http4sEndpointIO = programmingLang match { + case "java" | "Java" => + DynamicUtil.createJavaHttp4sEndpoint(decodedMethodBody) match { + case Full(func) => func + case Failure(msg: String, exception: Box[Throwable], _) => + throw exception.getOrElse(new RuntimeException(msg)) + case _ => throw new RuntimeException("compiled code return nothing") + } + case _ /* "Scala" | "scala" | "" | null, default */ => + scalaPartialFunction + } + + // Unchanged Scala-template compile path, factored out so the `partialFunction` match above stays + // readable. Only evaluated for Scala-language docs (the default) — Java-language docs never + // touch this, so example/response-body JValues that don't fit the Scala case-class generator + // (irrelevant for Java, since it doesn't use RequestRootJsonClass/ResponseRootJsonClass) are a + // non-issue there. + private def scalaPartialFunction: Http4sEndpointIO = { //If the requestBody is PrimaryDataBody, return None. otherwise, return the exampleRequestBody:Option[JValue] // In side OBP resourceDoc, requestBody and successResponse must be Product type， diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala index 8e6350494b..71b33776fd 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala @@ -49,7 +49,7 @@ object DynamicResourceDocsEndpointGroup extends EndpointGroup with code.util.Hel * */ private val toResourceDoc: JsonDynamicResourceDoc => ResourceDoc = { dynamicDoc => - val compiledObjects = CompiledObjects(dynamicDoc.exampleRequestBody, dynamicDoc.successResponseBody, dynamicDoc.methodBody) + val compiledObjects = CompiledObjects(dynamicDoc.exampleRequestBody, dynamicDoc.successResponseBody, dynamicDoc.methodBody, dynamicDoc.programmingLang) ResourceDoc( // partialFunction is a no-op stub — the runtime dispatch uses the native handler in // dynamicHttp4sFunction (the compiled artifact is OBPEndpointIO, not the Lift OBPEndpoint). diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 69ca8dea3c..6cbb891aed 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -44,6 +44,7 @@ object DynamicUtil extends MdcLoggable{ val toolBox: ToolBox[universe.type] = runtimeMirror(getClass.getClassLoader).mkToolBox() private val memoClassPool = new Memo[ClassLoader, ClassPool] + private val memoJavaHttp4sEndpoint = new Memo[String, Box[code.api.util.APIUtil.Http4sEndpointIO]] private def getClassPool(classLoader: ClassLoader) = memoClassPool.memoize(classLoader){ val cp = ClassPool.getDefault @@ -543,4 +544,135 @@ object DynamicUtil extends MdcLoggable{ } } } + + /** + * Converts a plain value returned by a compiled Java `method_body` into a JValue, for endpoints + * where json4s' `Extraction.decompose` cannot help: it works by Scala-case-class/collection + * reflection, so a `java.util.Map`/`java.util.List` returned from Java decomposes to `{}`/`[]` + * (its entries are invisible to Scala reflection) rather than throwing — a silent data-loss bug, + * not a compile or runtime error, so it only surfaces as an empty response body. Recurses through + * the Java collection types directly; anything else (including a Scala case class constructed + * from Java, as ConnectorMethod's Java example does) falls back to Extraction.decompose. + */ + private def javaValueToJValue(value: Any): JValue = { + import scala.jdk.CollectionConverters._ + value match { + case null => JNull + case jv: JValue => jv + case m: java.util.Map[_, _] => + JObject(m.asScala.toList.map { case (k, v) => (String.valueOf(k), javaValueToJValue(v)) }) + case l: java.util.List[_] => + JArray(l.asScala.toList.map(javaValueToJValue)) + case s: String => JString(s) + case b: java.lang.Boolean => JBool(b) + case i: java.lang.Integer => JInt(BigInt(i.intValue())) + case l: java.lang.Long => JInt(BigInt(l.longValue())) + case d: java.lang.Double => JDouble(d.doubleValue()) + case f: java.lang.Float => JDouble(f.doubleValue()) + case bd: java.math.BigDecimal => JDecimal(BigDecimal(bd)) + case other => Extraction.decompose(other)(CustomJsonFormats.formats) + } + } + + /** + * Compiles a Java `method_body` for a DynamicResourceDoc endpoint into a native + * `Http4sEndpointIO` (`PartialFunction[Request[IO], CallContext => IO[Response[IO]]]`), the same + * type the Scala template compiles to in DynamicEndpoints.CompiledObjects. + * + * Reuses the same JSR-223 "java" engine (backed by a real javax.tools.JavaCompiler via + * ch.obermuhlner:java-scriptengine — see createJavaFunction above) and the same + * package-uniquification trick, but — unlike createJavaFunction, whose DynamicFunction shape is + * specific to the ConnectorMethod feature — wraps the compiled function in a hand-written + * Http4sEndpointIO here in Scala. The Java method_body never has to construct cats.effect.IO, + * org.http4s.Response, or a Scala PartialFunction: it only ever returns a plain Java object + * (Map/List/String/number/boolean/etc.), which this adapter serializes via javaValueToJValue + * above (NOT Extraction.decompose directly — see that method's doc comment for why). + * + * Java-side convention (identical to the existing ConnectorMethod convention): the pasted class + * implements java.util.function.Supplier>. The + * compiled function is invoked with: + * args(0) = the raw request body (String, or null if the request had none) + * args(1) = path params (java.util.Map) + * args(2) = the CallContext (present whenever this endpoint is actually being served) + * mirroring createJavaFunction's own `func(args ++ cc)` call (line above): appending an + * Option[CallContext] via `++` appends its *contents* (0 or 1 raw CallContext), not the Option + * wrapper itself, so Java reads args[2] directly as a CallContext, no unwrapping needed. + * + * Unlike createJavaFunction, this validates the actual compiled Java class (not just its Scala + * wrapper) against `dynamic_code_compile_validate_dependencies`/`dynamic_code_compile_validate_enable`. + * CompiledObjects.validateDependency() (called by the ResourceDoc-creation flow) only ever sees + * `this.partialFunction` — the hand-written Http4sEndpointIO below — whose own bytecode just + * calls `java.util.function.Function.apply`, a non-restricted type; it can't see what the pasted + * Java class does inside apply(Object[]). Worse, `func` itself (the Function returned by the + * pasted class's get()) is commonly a method reference (`this::apply`), which the JVM + * materialises as a synthetic lambda class whose bytecode is just a delegating call — validating + * `func.getClass` would be equally blind. So we go through the JSR-223 Compilable API directly + * (JavaScriptEngine implements it) instead of plain eval(), to get the real top-level compiled + * class/instance (JavaCompiledScript.getCompiledClass/getCompiledInstance) and validate that + * before the function is ever returned or invoked. + */ + def createJavaHttp4sEndpoint(methodBody: String): Box[code.api.util.APIUtil.Http4sEndpointIO] = + if (!dynamicCodeExecutionEnabled) Failure(ErrorMessages.DynamicCodeExecutionDisabled) + else memoJavaHttp4sEndpoint.memoize("java-http4s-endpoint:" + methodBody) { + import cats.effect.IO + import code.api.util.APIUtil.Http4sEndpointIO + import com.openbankproject.commons.ExecutionContext.Implicits.global + import com.openbankproject.commons.util.JsonAliases.compactRender + import org.http4s.headers.`Content-Type` + import org.http4s.dsl.io._ + import org.http4s.{MediaType, Request, Response} + + import scala.jdk.CollectionConverters._ + + Box tryo { + val packageExp = UUID.randomUUID().toString.replaceAll("^|-", "_") + val packageMatcher = Pattern.compile("""(?m)^\s*package\s+\S+?\s*;""").matcher(methodBody) + + val javaCode = s"""package code.api.util.dynamic.${packageExp}; + |${packageMatcher.replaceFirst("")} + |""".stripMargin + + // Real compile happens here (javax.tools.JavaCompiler via the JSR-223 "java" engine) — + // any Java syntax/type error surfaces as an exception, caught by the outer `Box tryo`. + val compiledScript = javaEngine.asInstanceOf[javax.script.Compilable].compile(javaCode) + .asInstanceOf[ch.obermuhlner.scriptengine.java.JavaCompiledScript] + + // Validate the real compiled Supplier class before it's ever invoked — see the doc comment + // above for why this must run against getCompiledInstance, not `func`/`this.partialFunction`. + Validation.validateDependency(compiledScript.getCompiledInstance) + + val func = compiledScript.eval().asInstanceOf[java.util.function.Function[Array[AnyRef], Any]] + val jsonContentType = `Content-Type`(MediaType.application.json) + + new Http4sEndpointIO { + override def isDefinedAt(req: Request[IO]): Boolean = true + + override def apply(req: Request[IO]): CallContext => IO[Response[IO]] = { cc => + val pathParams: java.util.Map[String, String] = cc.resourceDocument + .map(_.getPathParams(req.uri.path.segments.toList.map(_.encoded))) + .getOrElse(Map.empty[String, String]) + .asJava + + val valueIO: IO[Any] = IO.fromFuture(IO { + Future { + val args: Array[AnyRef] = Array(cc.httpBody.orNull, pathParams) + func(args ++ Some(cc)) + } + }) + + valueIO.flatMap { value => + Ok(compactRender(javaValueToJValue(value)), jsonContentType) + }.handleErrorWith { e => + logger.warn(s"createJavaHttp4sEndpoint: Java method_body threw", e) + InternalServerError( + compactRender(Extraction.decompose( + Map("code" -> 500, "message" -> s"OBP-50000: Unknown Error. ${e.getMessage}") + )(CustomJsonFormats.formats)), + jsonContentType + ) + } + } + } + } + } } diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala index d821649e0d..8aaa8cc810 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala @@ -9374,12 +9374,19 @@ object Http4s400 { case _ => true } } + // Fail fast with a clean 400 before attempting compilation, rather than surfacing an + // unsupported programming_lang only as a generic DynamicCodeCompileFail. + _ <- code.util.Helper.booleanToFuture( + s"""$DynamicCodeLangNotSupport programming_lang ${body.programmingLang}, currently supported languages: Scala, Java""", + cc = Some(cc)) { + Set("", "scala", "Scala", "java", "Java").contains(body.programmingLang) + } } yield () } private def compileDynamicResourceDoc(body: JsonDynamicResourceDoc, cc: CallContext): Unit = { try { - CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody).validateDependency() + CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang).validateDependency() } catch { case e: JsonResponseException => throw e case e: Exception => diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala index a96c164350..b46485383e 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala @@ -25,6 +25,9 @@ class DynamicResourceDoc extends LongKeyedMapper[DynamicResourceDoc] with IdPK w object Tags extends MappedString(this, 255) object Roles extends MappedString(this, 255) object MethodBody extends MappedText(this) + // Source language of MethodBody: "Scala" (default) or "Java". Mirrors DynamicMessageDoc.Lang / + // ConnectorMethod.programmingLang — same field name/width convention, see DynamicEndpoints. + object Lang extends MappedString(this, 50) // Provenance: who created / last updated this runtime-compiled endpoint, and a SHA-256 of the // (decoded) method body so tampering / drift is detectable. Set server-side from the CallContext // user — never from the request body. createdAt / updatedAt come from the CreatedUpdated trait. @@ -50,7 +53,8 @@ object DynamicResourceDoc extends DynamicResourceDoc with LongKeyedMetaMapper[Dy successResponseBody = Option(dynamicResourceDoc.SuccessResponseBody.get).filter(StringUtils.isNotBlank).map(json.parse), errorResponseBodies = dynamicResourceDoc.ErrorResponseBodies.get, tags = dynamicResourceDoc.Tags.get, - roles = dynamicResourceDoc.Roles.get + roles = dynamicResourceDoc.Roles.get, + programmingLang = dynamicResourceDoc.Lang.get ) } diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala index 60e94a243a..aca89c7f76 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala @@ -33,7 +33,12 @@ case class JsonDynamicResourceDoc( successResponseBody: Option[JValue], errorResponseBodies: String, tags: String, - roles: String + roles: String, + // Source language of methodBody: "Scala" (default) or "Java". Mirrors + // JsonConnectorMethod.programmingLang / JsonDynamicMessageDoc.programmingLang. Appended last + // (not inserted alphabetically) so existing named-arg call sites and JSON payloads that predate + // this field keep compiling/deserializing unchanged. + programmingLang: String = "Scala" ) extends JsonFieldReName { def decodedMethodBody: String = URLDecoder.decode(methodBody, "UTF-8") } diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala index 47be5d0442..7b8d25a700 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala @@ -82,6 +82,7 @@ object MappedDynamicResourceDocProvider extends DynamicResourceDocProvider { .Tags(entity.tags) .Roles(entity.roles) .MethodBody(entity.methodBody) + .Lang(entity.programmingLang) // provenance is set here from the authenticated user + computed hash, not from `entity` .CreatedByUserId(createdByUserId.getOrElse(null)) .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) @@ -107,6 +108,7 @@ object MappedDynamicResourceDocProvider extends DynamicResourceDocProvider { .Tags(entity.tags) .Roles(entity.roles) .MethodBody(entity.methodBody) + .Lang(entity.programmingLang) // CreatedByUserId is left untouched; record who last changed the code + refresh the hash .UpdatedByUserId(updatedByUserId.getOrElse(null)) .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) diff --git a/obp-api/src/test/resources/frozen_type_meta_data b/obp-api/src/test/resources/frozen_type_meta_data index a15d8f19007c90d68212eb29be282ccc4714a6df..a49f5f72b3abe876dbae42c5566d78b45b1f5810 100644 GIT binary patch delta 180 zcmaF4oNMWFt_?Gd7=5Q}`!ULG-eHu>KmGb-M*Zm*#F@A^+c_;XW)z#8cvg6NdZs54pN zsOaSLRVvf}@G?nFeiylM`=lF;<>HKD+qph4GI27_pL{W2a=Y43#t4JyPePeYr~9!n Ya&K4GVtQl%H11#>qwsdgXr>Dm0IgL*BLDyZ delta 202 zcmZ3woa^Ost_?Gd82zUE_%X_D-eHu>KRMfyZ@R88BlqTbr-jCh!kZb(k`x(*Co`65 zZ|BShjorWDGHwynYeaWUt%2Y+jBIAkwk7C+q0RZ** BNhtsT diff --git a/obp-api/src/test/scala/code/api/util/DynamicUtilJavaHttp4sEndpointTest.scala b/obp-api/src/test/scala/code/api/util/DynamicUtilJavaHttp4sEndpointTest.scala new file mode 100644 index 0000000000..ab984a317b --- /dev/null +++ b/obp-api/src/test/scala/code/api/util/DynamicUtilJavaHttp4sEndpointTest.scala @@ -0,0 +1,109 @@ +package code.api.util + +import cats.effect.IO +import cats.effect.unsafe.implicits.global +import org.http4s.{Method, Request, Uri} +import org.json4s.native.JsonMethods.parse +import org.scalatest.{FeatureSpec, GivenWhenThen, Matchers} + +/** + * Focused unit test for DynamicUtil.createJavaHttp4sEndpoint, isolated from the full + * register -> role-check -> HTTP-dispatch round trip (see DynamicResourceDocJavaTest for that). + * Exercises the adapter directly: compiled Java Supplier> -> + * Http4sEndpointIO.apply(Request[IO]) -> CallContext => IO[Response[IO]]. + */ +class DynamicUtilJavaHttp4sEndpointTest extends FeatureSpec with Matchers with GivenWhenThen { + + private val echoMethodBody = + """package code.api.util.dynamic; + | + |import code.api.util.CallContext; + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaHttp4sEndpointUnitTest implements Supplier> { + | private Object apply(Object[] args) { + | String rawBody = (String) args[0]; + | @SuppressWarnings("unchecked") + | Map pathParams = (Map) args[1]; + | CallContext cc = (CallContext) args[2]; + | + | Map response = new LinkedHashMap<>(); + | response.put("echoed_body", rawBody); + | response.put("path_param_count", pathParams.size()); + | response.put("correlation_id", cc.correlationId()); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + feature("DynamicUtil.createJavaHttp4sEndpoint compiles a Java method_body into a native Http4sEndpointIO") { + + scenario("the compiled endpoint reads args(0)/args(1)/args(2) and serves 200 JSON") { + Given("a Java method_body compiled via createJavaHttp4sEndpoint") + val endpoint = DynamicUtil.createJavaHttp4sEndpoint(echoMethodBody).openOrThrowException("compilation failed") + + When("the compiled endpoint handles a request carrying a body, no path params, and a CallContext") + val req = Request[IO](method = Method.POST, uri = Uri.unsafeFromString("/test")) + val cc = CallContext(httpBody = Some("""{"hello":"world"}"""), correlationId = "test-correlation-id") + val resp = endpoint.apply(req)(cc).unsafeRunSync() + + Then("the response is 200 and echoes the body, the (empty) path params, and the CallContext's correlationId") + resp.status.code should equal(200) + val bodyString = resp.body.through(fs2.text.utf8.decode).compile.string.unsafeRunSync() + val json = parse(bodyString) + (json \ "echoed_body").values should equal("""{"hello":"world"}""") + (json \ "path_param_count").values should equal(BigInt(0)) + (json \ "correlation_id").values should equal("test-correlation-id") + } + + scenario("a Java compile error is reported as a Box Failure, not a thrown exception") { + Given("a method_body that is not valid Java") + val badMethodBody = "this is not valid java at all" + + When("we try to compile it") + val result = DynamicUtil.createJavaHttp4sEndpoint(badMethodBody) + + Then("compilation fails gracefully") + result.isDefined should equal(false) + } + + scenario("a Java method_body that throws at runtime is recovered as a 500, not an uncaught exception") { + Given("a Java method_body whose apply() throws") + val throwingMethodBody = + """package code.api.util.dynamic; + | + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaHttp4sEndpointThrowingTest implements Supplier> { + | private Object apply(Object[] args) { + | throw new RuntimeException("boom"); + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + val endpoint = DynamicUtil.createJavaHttp4sEndpoint(throwingMethodBody).openOrThrowException("compilation failed") + + When("the compiled endpoint is invoked") + val req = Request[IO](method = Method.POST, uri = Uri.unsafeFromString("/test")) + val resp = endpoint.apply(req)(CallContext()).unsafeRunSync() + + Then("the response is 500 rather than the IO failing") + resp.status.code should equal(500) + val bodyString = resp.body.through(fs2.text.utf8.decode).compile.string.unsafeRunSync() + bodyString should include("boom") + } + } +} diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala new file mode 100644 index 0000000000..34cb2cedee --- /dev/null +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala @@ -0,0 +1,151 @@ +package code.api.v4_0_0 + +import org.json4s._ +import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.util.ApiRole +import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, DynamicCodeLangNotSupport, UserHasMissingRoles} +import code.dynamicResourceDoc.JsonDynamicResourceDoc +import code.entitlement.Entitlement +import com.openbankproject.commons.model.ErrorMessage +import com.openbankproject.commons.util.json +import org.json4s.native.JsonMethods.{compact, parse => parseJson, render} +import org.json4s.native.Serialization.write + +/** + * Java-language coverage for the DynamicResourceDoc runtime-compilation mechanism. + * DynamicResourceDocTest.scala covers the (unchanged) Scala-language path end-to-end; these + * scenarios exercise the new `programming_lang = "Java"` dispatch added to + * DynamicEndpoints.CompiledObjects / DynamicUtil.createJavaHttp4sEndpoint, plus the + * backward-compat and unsupported-language guards added alongside it. + */ +class DynamicResourceDocJavaTest extends V400ServerSetup { + + // Java-side convention: the pasted class implements Supplier>. + // args(0) = raw request body (String, or null), args(1) = path params (java.util.Map), + // args(2) = the CallContext. See DynamicUtil.createJavaHttp4sEndpoint's doc comment. + private def javaRoleTestMethodBody: String = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaResourceDocRoleTest implements Supplier> { + | private Object apply(Object[] args) { + | String rawBody = (String) args[0]; + | @SuppressWarnings("unchecked") + | Map pathParams = (Map) args[1]; + | String myUserId = pathParams.get("MY_USER_ID"); + | + | Map response = new LinkedHashMap<>(); + | response.put("user_id_from_path", myUserId + "_from_path"); + | response.put("received_body", rawBody); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + feature("Native execution of a runtime-compiled dynamic resource doc with a Java method_body") { + + scenario("Create a role-gated Java-language dynamic resource doc and verify 401 / 403 / 200") { + val dynamicRole = "CanCallJavaPieceCRoleTest" + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + When("We create a Java-language dynamic resource doc gated by that role") + val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = dynamicRole, + partialFunctionName = "javaPieceCRoleTest", + requestUrl = "/my_java_role_user/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(javaRoleTestMethodBody, "UTF-8"), + programmingLang = "Java" + ) + val createResp = makePostRequest(createReq, write(doc)) + Then("We should get a 201") + createResp.code should equal(201) + createResp.body.extract[JsonDynamicResourceDoc].programmingLang should equal("Java") + + val callUrl = dynamicEndpoint_Request / "dynamic-resource-doc" / "my_java_role_user" / "user-1" + val body = """{"name":"Jhon","age":12,"hobby":["coding"]}""" + + Then("calling without authentication returns 401") + val resp401 = makePostRequest(callUrl.POST, body) + resp401.code should equal(401) + resp401.body.extract[ErrorMessage].message should include(AuthenticatedUserIsRequired) + + Then("calling authenticated but without the role returns 403") + val resp403 = makePostRequest(callUrl.POST <@ (user1), body) + resp403.code should equal(403) + resp403.body.extract[ErrorMessage].message should include(UserHasMissingRoles) + + Then("granting the role makes the call succeed (200), served by the compiled Java class") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, dynamicRole) + val resp200 = makePostRequest(callUrl.POST <@ (user1), body) + resp200.code should equal(200) + val rendered = json.compactRender(resp200.body) + rendered should include("user-1_from_path") + rendered should include("Jhon") + } + + scenario("Reject an unsupported programming_lang before attempting compilation") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + When("We create a dynamic resource doc with an unsupported programming_lang") + val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + partialFunctionName = "unsupportedLangTest", + requestUrl = "/unsupported_lang_test/MY_USER_ID", + programmingLang = "Python" + ) + val resp = makePostRequest(createReq, write(doc)) + + Then("We should get a 400 DynamicCodeLangNotSupport, not a compile-failure error") + resp.code should equal(400) + resp.body.extract[ErrorMessage].message should include(DynamicCodeLangNotSupport) + } + + scenario("Backward compatibility: a request body with programming_lang entirely omitted still creates a Scala-language doc") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + When("We create a dynamic resource doc from a JSON payload that predates the programming_lang field") + val posted = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "preExistingClientTest", + requestUrl = "/pre_existing_client_test/MY_USER_ID" + ) + // Simulate an old client payload: strip programming_lang out entirely rather than relying on + // Serialization.write, which always emits every case-class field (default or not). + val fullJson = parseJson(write(posted)) + val withoutLang = fullJson.removeField { case (name, _) => name == "programming_lang" } + val requestBodyStr = compact(render(withoutLang)) + requestBodyStr should not include "programming_lang" + + val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val createResp = makePostRequest(createReq, requestBodyStr) + + Then("We should get a 201 and the stored/served doc defaults to the Scala language") + createResp.code should equal(201) + createResp.body.extract[JsonDynamicResourceDoc].programmingLang should equal("Scala") + + Then("calling the endpoint still compiles and serves via the (unchanged) Scala template path") + val callReq = (dynamicEndpoint_Request / "dynamic-resource-doc" / "pre_existing_client_test" / "user-1").POST <@ (user1) + val callResp = makePostRequest(callReq, """{"name":"Jhon","age":12,"hobby":["coding"]}""") + callResp.code should equal(200) + val rendered = json.compactRender(callResp.body) + rendered should include("user-1_from_path") + rendered should include("Jhon") + } + } +} From 69eaa44fddec063780b6a59a8140529b563ccaf7 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 16:43:36 +0200 Subject: [PATCH 32/59] fix: thread programming_lang into dynamic-resource-doc validate endpoint POST /management/dynamic-resource-docs/validate constructed CompiledObjects without the doc's programming_lang, so validating a Java-language method_body silently tried to compile it as Scala and always reported a (misleading) CompilationError, and the success message hardcoded "valid Scala" regardless of the actual language. --- obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index 3b91ef8fff..aeae1a0f39 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -4358,10 +4358,10 @@ object Http4s600 { } } yield try { code.api.dynamic.endpoint.helper.CompiledObjects( - body.exampleRequestBody, body.successResponseBody, body.methodBody).validateDependency() + body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang).validateDependency() ValidateDynamicResourceDocSuccessJsonV600( valid = true, - message = "Dynamic Resource Doc method body is valid Scala and uses allowed dependencies.") + message = s"Dynamic Resource Doc method body is valid ${body.programmingLang} and uses allowed dependencies.") } catch { case e: code.api.JsonResponseException => val errorText = e.jsonResponse match { From 4b469ef5053098c6b01812e4d812840c72c6450a Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 16:51:32 +0200 Subject: [PATCH 33/59] refactor: derive resource-doc dispatch and the global operation-id union from one registry Introduce ResourceDocRegistry as the single source of truth for "which resource docs does version X serve", replacing two independently hand-maintained registries: ResourceDocsAPIMethods.getResourceDocsList (the per-version dispatcher used by /resource-docs/{VERSION}/... and API Explorer) and APIUtil.allStaticResourceDocs (the union used wherever an operation id must be resolved). These drifted three times by hand -- Berlin Group v2, v7-only operation ids, and the Berlin Group v1.3 alias all had to be independently added to both places, and were each missed at least once. Deriving both from one registry map makes that class of drift structurally impossible going forward. Http4sBGv2 becomes a ScannedApis registrant (its apiVersion is ConstantsBG.berlinGroupVersion2), so it is now fully convention-driven like the other Berlin Group / UK Open Banking standards and needs no hand-maintained entry in the registry or a special case in ApiVersionUtils.valueOf. The global union is now deduped by operationId -- the underlying per-version buffers legitimately overlap (each OBP-standard aggregation repeats every older version's docs), and consumers only ever .find or build a lookup map from the result. ResourceDocRegistryParityTest is rewritten to iterate the registry itself rather than a hand-typed list of standards, so a future standard reachable by the dispatcher is covered by construction and the test's job narrows to catching an accidental regression back to two independently maintained registries. Verified live against a running instance, before and after: BGv2 and Berlin Group v1.3 alias operation ids both still resolve through POST /my/api-collections/{name}/api-collection-endpoints. Full local suite: 3582 tests, 0 failures. --- .../ResourceDocsAPIMethods.scala | 27 +----- .../code/api/berlin/group/v2/Http4sBGv2.scala | 10 ++- .../main/scala/code/api/util/APIUtil.scala | 39 ++------- .../scala/code/api/util/ApiVersionUtils.scala | 3 - .../code/api/util/ResourceDocRegistry.scala | 82 +++++++++++++++++++ .../util/ResourceDocRegistryParityTest.scala | 62 +++++++------- 6 files changed, 134 insertions(+), 89 deletions(-) create mode 100644 obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala index 8263d92d79..a35a0483a4 100644 --- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala +++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala @@ -3,8 +3,6 @@ package code.api.ResourceDocs1_4_0 import code.api.Constant.{GET_DYNAMIC_RESOURCE_DOCS_TTL, GET_STATIC_RESOURCE_DOCS_TTL, HostName, PARAM_LOCALE} import code.api.OBPRestHelper import code.api.cache.Caching -import code.api.dynamic.endpoint.OBPAPIDynamicEndpoint -import code.api.dynamic.entity.OBPAPIDynamicEntity import code.api.util.APIUtil._ import code.api.util.ApiRole.{canReadDynamicResourceDocsAtOneBank, canReadResourceDoc} import code.api.util.ApiTag._ @@ -22,7 +20,6 @@ import code.api.v4_0_0.{APIMethods400, OBPAPI4_0_0} import code.api.v5_0_0.OBPAPI5_0_0 import code.api.v5_1_0.OBPAPI5_1_0 import code.api.v6_0_0.OBPAPI6_0_0 -import code.api.berlin.group.ConstantsBG import code.apicollectionendpoint.MappedApiCollectionEndpointsProvider import code.util.Helper import code.util.Helper.{MdcLoggable, ObpS, SILENCE_IS_GOLDEN} @@ -331,27 +328,9 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth logger.debug(s"getResourceDocsList says requestedApiVersion is $requestedApiVersion") - val resourceDocs = requestedApiVersion match { - case ApiVersion.v7_0_0 => code.api.v7_0_0.Http4s700.allResourceDocs // Use aggregated docs for v7.0.0 - case ConstantsBG.`berlinGroupVersion1` => code.api.berlin.group.v1_3.Http4sBGv13.resourceDocs - case ConstantsBG.`berlinGroupVersion2` => code.api.berlin.group.v2.Http4sBGv2.resourceDocs - case ApiVersion.v6_0_0 => code.api.util.http4s.Http4sResourceDocAggregation.v600 - case ApiVersion.v5_1_0 => code.api.util.http4s.Http4sResourceDocAggregation.v510 - case ApiVersion.v5_0_0 => code.api.util.http4s.Http4sResourceDocAggregation.v500 - case ApiVersion.v4_0_0 => code.api.util.http4s.Http4sResourceDocAggregation.v400 - case ApiVersion.v3_1_0 => code.api.util.http4s.Http4sResourceDocAggregation.v310 - case ApiVersion.v3_0_0 => code.api.util.http4s.Http4sResourceDocAggregation.v300 - case ApiVersion.v2_2_0 => code.api.util.http4s.Http4sResourceDocAggregation.v220 - case ApiVersion.v2_1_0 => code.api.util.http4s.Http4sResourceDocAggregation.v210 - case ApiVersion.v2_0_0 => code.api.util.http4s.Http4sResourceDocAggregation.v200 - case ApiVersion.v1_4_0 => code.api.util.http4s.Http4sResourceDocAggregation.v140 - case ApiVersion.v1_3_0 => code.api.util.http4s.Http4sResourceDocAggregation.v130 - case ApiVersion.v1_2_1 => code.api.util.http4s.Http4sResourceDocAggregation.v121 - case ApiVersion.`dynamic-endpoint` => OBPAPIDynamicEndpoint.allResourceDocs - case ApiVersion.`dynamic-entity` => OBPAPIDynamicEntity.allResourceDocs - case version: ScannedApiVersion => ScannedApis.versionMapScannedApis.get(version).map(_.allResourceDocs).getOrElse(ArrayBuffer.empty[ResourceDoc]) - case _ => ArrayBuffer.empty[ResourceDoc] - } + // ResourceDocRegistry is the single source of truth for both this per-version dispatch and + // APIUtil.allStaticResourceDocs' global operation-id union -- see that object's doc comment. + val resourceDocs = ResourceDocRegistry.docsFor(requestedApiVersion) logger.debug(s"There are ${resourceDocs.length} resource docs available to $requestedApiVersion") diff --git a/obp-api/src/main/scala/code/api/berlin/group/v2/Http4sBGv2.scala b/obp-api/src/main/scala/code/api/berlin/group/v2/Http4sBGv2.scala index 9e2d1ce640..b698cb0dc4 100644 --- a/obp-api/src/main/scala/code/api/berlin/group/v2/Http4sBGv2.scala +++ b/obp-api/src/main/scala/code/api/berlin/group/v2/Http4sBGv2.scala @@ -4,23 +4,31 @@ import cats.data.{Kleisli, OptionT} import cats.effect._ import code.api.berlin.group.ConstantsBG import code.api.util.APIUtil.ResourceDoc +import code.api.util.ScannedApis import code.api.util.http4s.ResourceDocMiddleware import code.util.Helper.MdcLoggable +import com.openbankproject.commons.util.ScannedApiVersion import org.http4s._ import scala.collection.mutable.ArrayBuffer -object Http4sBGv2 extends MdcLoggable { +object Http4sBGv2 extends MdcLoggable with ScannedApis { type HttpF[A] = OptionT[IO, A] val implementedInApiVersion = ConstantsBG.berlinGroupVersion2 + // ScannedApis discovery marker: makes BGv2 convention-driven like the other Berlin Group / + // UK Open Banking standards, so ResourceDocRegistry picks it up without a hand-maintained entry. + override val apiVersion: ScannedApiVersion = implementedInApiVersion + val resourceDocs: ArrayBuffer[ResourceDoc] = Http4sBGv2AIS.resourceDocs ++ Http4sBGv2PIS.resourceDocs ++ Http4sBGv2PIIS.resourceDocs + override val allResourceDocs: ArrayBuffer[ResourceDoc] = resourceDocs + val allRoutes: HttpRoutes[IO] = Kleisli[HttpF, Request[IO], Response[IO]] { req => Http4sBGv2AIS.routes(req) .orElse(Http4sBGv2PIS.routes(req)) diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index b3d685f30b..4d9f61bcff 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -32,9 +32,6 @@ import cats.effect.IO import code.abacrule.AbacRuleEngine import code.accountholders.AccountHolders import code.api.Constant._ -import code.api.UKOpenBanking.v2_0_0.OBP_UKOpenBanking_200 -import code.api.UKOpenBanking.v3_1_0.OBP_UKOpenBanking_310 -import code.api.UKOpenBanking.v4_0_1.OBP_UKOpenBanking_401 import code.api._ import code.api.berlin.group.ConstantsBG import code.api.berlin.group.v1_3.JSONFactory_BERLIN_GROUP_1_3.{ErrorMessageBG, ErrorMessagesBG} @@ -4874,35 +4871,13 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ val allowedAnswerTransactionRequestChallengeAttempts = APIUtil.getPropsAsIntValue("answer_transactionRequest_challenge_allowed_attempts").openOr(3) - // Base is the v7 aggregation — the newest OBP-standard surface, which already contains the - // v6.0.0-and-older aggregation plus the v7-only endpoints (deduped by URL/method). Basing on - // the v6 aggregation silently excluded v7-only operation ids from everything that resolves - // operation ids through this list (api-collection endpoint validation, top-apis lookups, ...). - // ResourceDocRegistryParityTest pins the invariant that every per-standard surface the - // resource-docs dispatcher can serve is contained here. - lazy val allStaticResourceDocs = (code.api.v7_0_0.Http4s700.allResourceDocs - ++ OBP_UKOpenBanking_200.allResourceDocs - ++ OBP_UKOpenBanking_310.allResourceDocs - ++ OBP_UKOpenBanking_401.allResourceDocs - // Commented out: Lift endpoints migrated off / removed (Polish, STET, AUOpenBanking, MxOF/CNBV9, BahrainOBF) - // ++ code.api.Polish.v2_1_1_1.OBP_PAPI_2_1_1_1.allResourceDocs - // ++ code.api.STET.v1_4.OBP_STET_1_4.allResourceDocs - // ++ code.api.AUOpenBanking.v1_0_0.ApiCollector.allResourceDocs - // ++ code.api.MxOF.CNBV9_1_0_0.allResourceDocs - // ++ code.api.MxOF.OBP_MXOF_1_0_0.allResourceDocs - // ++ code.api.BahrainOBF.v1_0_0.ApiCollector.allResourceDocs - ++ code.api.berlin.group.v1_3.OBP_BERLIN_GROUP_1_3.allResourceDocs - // The BG v1.3 alias (active only when berlin_group_v1_3_alias_path is set) is served by - // the resource-docs dispatcher via its ScannedApis registration, but its docs carry their - // own operation ids (re-derived from the alias version), so it hit the same - // getAllResourceDocs membership gap as BGv2 below. Empty when the prop is unset, so this - // is a no-op on instances without the alias configured. - ++ code.api.berlin.group.v1_3.OBP_BERLIN_GROUP_1_3_Alias.allResourceDocs - // BGv2 was missing here even though /resource-docs/BGv2 serves it, so a BGv2 operation id - // (e.g. BGv2-getAccountDetails) failed the getAllResourceDocs membership check that - // api-collection-endpoints (and anything else resolving operation ids) relies on. - ++ code.api.berlin.group.v2.Http4sBGv2.resourceDocs).toList - + // Delegates to ResourceDocRegistry, the single source of truth shared with the per-version + // resource-docs dispatcher (ResourceDocsAPIMethods.getResourceDocsList) -- see that object's + // doc comment for why the two used to drift and how deriving both from one registry fixes it. + // Kept under this name so existing call sites (Http4s400, Http4s600, JSONFactory6.0.0, ...) + // don't need to move. + lazy val allStaticResourceDocs: List[ResourceDoc] = ResourceDocRegistry.allStaticResourceDocs + def allDynamicResourceDocs= (DynamicEntityHelper.doc ++ DynamicEndpointHelper.doc ++ DynamicEndpoints.dynamicResourceDocs).toList def getAllResourceDocs = allStaticResourceDocs ++ allDynamicResourceDocs diff --git a/obp-api/src/main/scala/code/api/util/ApiVersionUtils.scala b/obp-api/src/main/scala/code/api/util/ApiVersionUtils.scala index cc2641c67f..d291130498 100644 --- a/obp-api/src/main/scala/code/api/util/ApiVersionUtils.scala +++ b/obp-api/src/main/scala/code/api/util/ApiVersionUtils.scala @@ -2,7 +2,6 @@ package code.api.util import com.openbankproject.commons.util.ApiVersion._ import com.openbankproject.commons.util.ScannedApiVersion -import code.api.berlin.group.ConstantsBG object ApiVersionUtils { @@ -23,7 +22,6 @@ object ApiVersionUtils { v7_0_0 :: `dynamic-endpoint` :: `dynamic-entity` :: - ConstantsBG.berlinGroupVersion2 :: scannedApis ).distinct @@ -48,7 +46,6 @@ object ApiVersionUtils { case v7_0_0.fullyQualifiedVersion | v7_0_0.apiShortVersion => v7_0_0 case `dynamic-endpoint`.fullyQualifiedVersion | `dynamic-endpoint`.apiShortVersion => `dynamic-endpoint` case `dynamic-entity`.fullyQualifiedVersion | `dynamic-entity`.apiShortVersion => `dynamic-entity` - case version if version == ConstantsBG.berlinGroupVersion2.fullyQualifiedVersion || version == ConstantsBG.berlinGroupVersion2.apiShortVersion => ConstantsBG.berlinGroupVersion2 case version if(scannedApis.map(_.fullyQualifiedVersion).contains(version)) =>scannedApis.filter(_.fullyQualifiedVersion==version).head case version if(scannedApis.map(_.apiShortVersion).contains(version)) diff --git a/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala b/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala new file mode 100644 index 0000000000..f0194d0aca --- /dev/null +++ b/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala @@ -0,0 +1,82 @@ +package code.api.util + +import code.api.berlin.group.ConstantsBG +import code.api.util.APIUtil.ResourceDoc +import com.openbankproject.commons.util.ApiVersion._ +import com.openbankproject.commons.util.ApiVersion + +import scala.collection.immutable.ListMap + +/** + * Single source of truth for "which resource docs does version X serve" — used both by the + * per-version resource-docs dispatcher (ResourceDocsAPIMethods.getResourceDocsList, i.e. + * /resource-docs/{VERSION}/... and API Explorer) and by the global operation-id union + * (allStaticResourceDocs / getAllResourceDocs, used wherever an operation id must be resolved: + * api-collection-endpoint creation, top-apis/popular-apis lookups, metrics, ...). + * + * These used to be two independently hand-maintained registries and drifted three times: Berlin + * Group v2 was served by the dispatcher but missing from the union (BGv2-getAccountDetails could + * not be added to an API collection), the union was based on the v6 aggregation excluding v7-only + * operation ids, and the Berlin Group v1.3 alias was missing from the union too. Deriving both + * from one `registry` map makes that class of drift structurally impossible: add a version once, + * both call sites see it. + * + * Rule for adding a new API standard: implement `with ScannedApis` (see that trait) and it is + * picked up automatically via the `scanned` half of `registry` below — no edit needed here. Only + * standards that cannot be discovered that way (or that need to override the source composing + * function, e.g. the cumulative per-version OBP-standard aggregations) need an `explicit` entry. + * + * Deliberately its own file/object, NOT a member of `APIUtil`: the Implementations* objects for + * each version re-enter `APIUtil` during their own initialization (prop lookups, etc.), so a + * strict `val` living inside `APIUtil` risks a class-init deadlock. Everything here stays `lazy` + * and is first touched at request/test time, well after Props and `ApiVersion.setUrlPrefix` have + * run in Boot. + */ +object ResourceDocRegistry { + + /** version -> that surface's docs. Thunks, not values: the ScannedApis-discovered arms are + * lazy vals themselves and the OBP-standard aggregations are cumulative lazy vals too — wrapping + * in a function defers evaluation to first use of THIS registry, not construction of the map. */ + lazy val registry: ListMap[ApiVersion, () => Seq[ResourceDoc]] = { + val explicit: ListMap[ApiVersion, () => Seq[ResourceDoc]] = ListMap( + v7_0_0 -> (() => code.api.v7_0_0.Http4s700.allResourceDocs.toSeq), + ConstantsBG.berlinGroupVersion1 -> (() => code.api.berlin.group.v1_3.Http4sBGv13.resourceDocs.toSeq), + v6_0_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v600.toSeq), + v5_1_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v510.toSeq), + v5_0_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v500.toSeq), + v4_0_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v400.toSeq), + v3_1_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v310.toSeq), + v3_0_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v300.toSeq), + v2_2_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v220.toSeq), + v2_1_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v210.toSeq), + v2_0_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v200.toSeq), + v1_4_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v140.toSeq), + v1_3_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v130.toSeq), + v1_2_1 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v121.toSeq), + `dynamic-endpoint` -> (() => code.api.dynamic.endpoint.OBPAPIDynamicEndpoint.allResourceDocs.toSeq), + `dynamic-entity` -> (() => code.api.dynamic.entity.OBPAPIDynamicEntity.allResourceDocs.toSeq) + // Berlin Group v2 is NOT listed here -- Http4sBGv2 is a ScannedApis registrant (its + // apiVersion is ConstantsBG.berlinGroupVersion2), so it is picked up by `scanned` below. + ) + // Every standard discovered via ScannedApis (UK OB 200/310/401, BG v1.3 canonical + alias, + // BG v2, and any future `with ScannedApis` standard). Explicit entries win on key collision -- + // BG v1.3 canonical is both explicit above and a registrant, same underlying buffer either way. + val scanned: Map[ApiVersion, () => Seq[ResourceDoc]] = ScannedApis.versionMapScannedApis.collect { + case (version, apis) if !explicit.contains(version) => version -> (() => apis.allResourceDocs.toSeq) + } + explicit ++ scanned + } + + /** What the per-version resource-docs dispatcher serves for this version (empty if unknown). */ + def docsFor(version: ApiVersion): Seq[ResourceDoc] = registry.get(version).map(_ ()).getOrElse(Nil) + + /** The global operation-id union. Excludes the dynamic arms: those are runtime-mutable (created/ + * deleted dynamic entities and endpoints) and are appended FRESH by APIUtil.getAllResourceDocs on + * every call -- caching them in this lazy union would serve stale dynamic docs. Deduped by + * operationId: the underlying per-version buffers legitimately overlap (each OBP-standard + * aggregation repeats every older version's docs), and consumers only ever `.find`/build a + * lookup map from this list, never rely on it containing duplicates. */ + lazy val allStaticResourceDocs: List[ResourceDoc] = + (registry - `dynamic-endpoint` - `dynamic-entity`) + .values.flatMap(_ ()).toList.distinctBy(_.operationId) +} diff --git a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala index fa0e25247a..341c65e122 100644 --- a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala +++ b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala @@ -1,22 +1,22 @@ package code.api.util import code.setup.ServerSetup +import com.openbankproject.commons.util.{ApiVersion, ScannedApiVersion} import org.scalatest.Tag /** - * Guards the invariant that APIUtil.getAllResourceDocs — the global operation-id - * registry used wherever an operation id must be resolved (api-collection endpoint - * validation, top-apis operation-id lookups, ...) — contains every per-standard - * resource-doc surface the resource-docs dispatcher can serve to API Explorer. + * Guards the invariant that APIUtil.getAllResourceDocs — the global operation-id registry used + * wherever an operation id must be resolved (api-collection endpoint validation, top-apis + * operation-id lookups, ...) — contains every per-standard resource-doc surface the resource-docs + * dispatcher can serve to API Explorer. * - * These are two parallel registries (ResourceDocsAPIMethods dispatches per - * standard/version; getAllResourceDocs aggregates them all), and they have drifted - * three times: Berlin Group v2 was served by the dispatcher but missing from the - * global registry (so BGv2-getAccountDetails could not be added to an API - * collection), the global registry was based on the v6 aggregation, excluding - * v7-only operation ids, and the Berlin Group v1.3 alias (active only when - * berlin_group_v1_3_alias_path is set) was missing too. When you add a NEW API - * standard, register its docs in BOTH places — and add its surface to this list. + * Both sides are now derived from the single ResourceDocRegistry.registry map, so this class of + * drift (which happened three times by hand: Berlin Group v2, v7-only operation ids, and the + * Berlin Group v1.3 alias) is structurally impossible going forward — see ResourceDocRegistry's + * doc comment. This test's job is narrower than it used to be: it iterates the registry itself + * (rather than a hand-typed list of standards) so it stays correct as standards are added or + * removed without needing an edit here, and it catches an accidental regression back to two + * independently hand-maintained registries. */ class ResourceDocRegistryParityTest extends ServerSetup { @@ -25,24 +25,28 @@ class ResourceDocRegistryParityTest extends ServerSetup { private lazy val allOperationIds: Set[String] = APIUtil.getAllResourceDocs.map(_.operationId).toSet - // optional=true surfaces are gated by a prop that is unset by default (test props - // included), so their operation-id list is legitimately empty in most environments -- - // the membership check still runs (trivially true when empty) but the non-empty - // assertion is skipped for them. - private lazy val surfaces: List[(String, Seq[String], Boolean)] = List( - ("OBP standard (v7 aggregation)", code.api.v7_0_0.Http4s700.allResourceDocs.map(_.operationId).toSeq, false), - ("Berlin Group v1.3", code.api.berlin.group.v1_3.Http4sBGv13.resourceDocs.map(_.operationId).toSeq, false), - ("Berlin Group v1.3 alias", code.api.berlin.group.v1_3.Http4sBGv13Alias.resourceDocs.map(_.operationId).toSeq, true), - ("Berlin Group v2", code.api.berlin.group.v2.Http4sBGv2.resourceDocs.map(_.operationId).toSeq, false), - ("UK Open Banking 2.0.0", code.api.UKOpenBanking.v2_0_0.OBP_UKOpenBanking_200.allResourceDocs.map(_.operationId).toSeq, false), - ("UK Open Banking 3.1.0", code.api.UKOpenBanking.v3_1_0.OBP_UKOpenBanking_310.allResourceDocs.map(_.operationId).toSeq, false), - ("UK Open Banking 4.0.1", code.api.UKOpenBanking.v4_0_1.OBP_UKOpenBanking_401.allResourceDocs.map(_.operationId).toSeq, false) - ) - - feature("getAllResourceDocs contains every per-standard resource-doc surface") { - surfaces.foreach { case (label, operationIds, optional) => + private def label(version: ApiVersion): String = version match { + case sv: ScannedApiVersion => sv.fullyQualifiedVersion + case other => other.toString + } + + // Dynamic arms (dynamic-endpoint / dynamic-entity) are excluded: they are runtime-mutable and + // ResourceDocRegistry.allStaticResourceDocs itself excludes them for the same reason (see its + // doc comment) -- APIUtil.getAllResourceDocs appends them fresh via allDynamicResourceDocs + // instead, so comparing them against this registry-derived snapshot would be meaningless. + private lazy val surfaces: List[(String, Seq[String])] = + (ResourceDocRegistry.registry - ApiVersion.`dynamic-endpoint` - ApiVersion.`dynamic-entity`) + .toList + .map { case (version, docsThunk) => (label(version), docsThunk().map(_.operationId)) } + + feature("getAllResourceDocs contains every per-standard resource-doc surface the dispatcher can serve") { + scenario("the registry itself is non-empty", RegistryParityTag) { + surfaces should not be empty + surfaces.exists(_._2.nonEmpty) shouldBe true + } + + surfaces.foreach { case (label, operationIds) => scenario(s"$label operation ids are all resolvable globally", RegistryParityTag) { - if (!optional) operationIds should not be empty val missing = operationIds.filterNot(allOperationIds.contains) withClue(s"$label operation ids missing from getAllResourceDocs: ${missing.take(10).mkString(", ")} ") { missing shouldBe empty From 5af4cb67ea7605439ecad87d6833f9c022574c54 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 17:07:09 +0200 Subject: [PATCH 34/59] test: pin BGv2-getAccountDetails as a resolvable api-collection-endpoint operation id Adds an HTTP-level regression test for the sandbox bug report this branch started from: creating an API collection endpoint with operation_id=BGv2-getAccountDetails now returns 201, alongside the existing coverage for OBPv6.0.0, UK Open Banking, and Berlin Group v1.3 operation ids in the same scenario. Previously the only regression guard for this exact operation id was the unit-level membership check in ResourceDocRegistryParityTest; this exercises the actual endpoint. --- .../v4_0_0/ApiCollectionEndpointTest.scala | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala index 5e57ea2a7e..755f1f1869 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala @@ -202,7 +202,28 @@ class ApiCollectionEndpointTest extends V400ServerSetup { val operationId= apiCollectionEndpoint.operation_id } - + + { + // Regression pin for the sandbox bug report: BGv2-getAccountDetails was served by the + // resource-docs dispatcher (/resource-docs/BGv2/obp) but missing from the global + // operation-id union getAllResourceDocs relies on, so this exact request used to fail + // with OBP-40048 Invalid operation_id. + Then(s"we test the $ApiEndpoint6- BGv2-getAccountDetails") + val requestApiCollectionEndpoint = (v4_0_0_Request / "my" / "api-collection-ids" / apiCollectionId / "api-collection-endpoints").POST <@ (user1) + + lazy val postApiCollectionEndpointJson = SwaggerDefinitionsJSON.postApiCollectionEndpointJson400.copy(operation_id="BGv2-getAccountDetails") + + val responseApiCollectionEndpointJson = makePostRequest(requestApiCollectionEndpoint, write(postApiCollectionEndpointJson)) + Then("We should get a 201") + responseApiCollectionEndpointJson.code should equal(201) + val apiCollectionEndpoint = responseApiCollectionEndpointJson.body.extract[ApiCollectionEndpointJson400] + + apiCollectionEndpoint.operation_id should be (postApiCollectionEndpointJson.operation_id) + apiCollectionEndpoint.api_collection_endpoint_id shouldNot be (null) + + val operationId= apiCollectionEndpoint.operation_id + } + { Then(s"we test the $ApiEndpoint7") val requestGet = (v4_0_0_Request / "my" / "api-collection-ids" / apiCollectionId / "api-collection-endpoints").GET <@ (user1) @@ -213,7 +234,7 @@ class ApiCollectionEndpointTest extends V400ServerSetup { val apiCollectionsJsonGet400 = responseGet.body.extract[ApiCollectionEndpointsJson400] - apiCollectionsJsonGet400.api_collection_endpoints.length should be (4) + apiCollectionsJsonGet400.api_collection_endpoints.length should be (5) } } } From 75dff0495da82640e46ab78e0b202acc48cf921e Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 17:19:10 +0200 Subject: [PATCH 35/59] test: extract duplicated dynamic-resource-docs request literal to a helper SonarCloud flagged the "dynamic-resource-docs" URL segment repeated three times across DynamicResourceDocJavaTest's three scenarios. --- .../code/api/v4_0_0/DynamicResourceDocJavaTest.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala index 34cb2cedee..8ce6a30af0 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala @@ -20,6 +20,8 @@ import org.json4s.native.Serialization.write */ class DynamicResourceDocJavaTest extends V400ServerSetup { + private def createDynamicResourceDocsRequest = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + // Java-side convention: the pasted class implements Supplier>. // args(0) = raw request body (String, or null), args(1) = path params (java.util.Map), // args(2) = the CallContext. See DynamicUtil.createJavaHttp4sEndpoint's doc comment. @@ -58,7 +60,7 @@ class DynamicResourceDocJavaTest extends V400ServerSetup { Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) When("We create a Java-language dynamic resource doc gated by that role") - val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val createReq = createDynamicResourceDocsRequest val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( dynamicResourceDocId = None, bankId = None, @@ -99,7 +101,7 @@ class DynamicResourceDocJavaTest extends V400ServerSetup { Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) When("We create a dynamic resource doc with an unsupported programming_lang") - val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val createReq = createDynamicResourceDocsRequest val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( dynamicResourceDocId = None, bankId = None, @@ -132,7 +134,7 @@ class DynamicResourceDocJavaTest extends V400ServerSetup { val requestBodyStr = compact(render(withoutLang)) requestBodyStr should not include "programming_lang" - val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val createReq = createDynamicResourceDocsRequest val createResp = makePostRequest(createReq, requestBodyStr) Then("We should get a 201 and the stored/served doc defaults to the Scala language") From cd1b548469ade7562896e2403f923f19e3ce0338 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 17:24:45 +0200 Subject: [PATCH 36/59] test: extract shared 401/403/200 role-gate assertion for dynamic resource docs DynamicResourceDocJavaTest's role-gated scenario duplicated the same 17-line 401/403/200 assertion block already in DynamicResourceDocTest, which SonarCloud's new-code duplication gate flagged. Both now share V400ServerSetup.assertRoleGated401Then403Then200. --- .../v4_0_0/DynamicResourceDocJavaTest.scala | 24 +++++------------ .../api/v4_0_0/DynamicResourceDocTest.scala | 20 +++----------- .../code/api/v4_0_0/V400ServerSetup.scala | 27 ++++++++++++++++++- 3 files changed, 36 insertions(+), 35 deletions(-) diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala index 8ce6a30af0..aa12c85bc8 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala @@ -3,7 +3,7 @@ package code.api.v4_0_0 import org.json4s._ import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON import code.api.util.ApiRole -import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, DynamicCodeLangNotSupport, UserHasMissingRoles} +import code.api.util.ErrorMessages.DynamicCodeLangNotSupport import code.dynamicResourceDoc.JsonDynamicResourceDoc import code.entitlement.Entitlement import com.openbankproject.commons.model.ErrorMessage @@ -78,23 +78,11 @@ class DynamicResourceDocJavaTest extends V400ServerSetup { val callUrl = dynamicEndpoint_Request / "dynamic-resource-doc" / "my_java_role_user" / "user-1" val body = """{"name":"Jhon","age":12,"hobby":["coding"]}""" - Then("calling without authentication returns 401") - val resp401 = makePostRequest(callUrl.POST, body) - resp401.code should equal(401) - resp401.body.extract[ErrorMessage].message should include(AuthenticatedUserIsRequired) - - Then("calling authenticated but without the role returns 403") - val resp403 = makePostRequest(callUrl.POST <@ (user1), body) - resp403.code should equal(403) - resp403.body.extract[ErrorMessage].message should include(UserHasMissingRoles) - - Then("granting the role makes the call succeed (200), served by the compiled Java class") - Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, dynamicRole) - val resp200 = makePostRequest(callUrl.POST <@ (user1), body) - resp200.code should equal(200) - val rendered = json.compactRender(resp200.body) - rendered should include("user-1_from_path") - rendered should include("Jhon") + assertRoleGated401Then403Then200(callUrl, body, dynamicRole) { resp200 => + val rendered = json.compactRender(resp200.body) + rendered should include("user-1_from_path") + rendered should include("Jhon") + } } scenario("Reject an unsupported programming_lang before attempting compilation") { diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala index a482557a18..ec60ccb3b9 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala @@ -29,7 +29,7 @@ import org.json4s._ import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON import code.api.util.APIUtil.OAuth._ import code.api.util.ApiRole._ -import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, DynamicResourceDocAlreadyExists, DynamicResourceDocNotFound, UserHasMissingRoles} +import code.api.util.ErrorMessages.{DynamicResourceDocAlreadyExists, DynamicResourceDocNotFound, UserHasMissingRoles} import code.api.util.ApiRole import code.api.v4_0_0.APIMethods400.Implementations4_0_0 import code.dynamicResourceDoc.JsonDynamicResourceDoc @@ -315,21 +315,9 @@ class DynamicResourceDocTest extends V400ServerSetup { val callUrl = dynamicEndpoint_Request / "dynamic-resource-doc" / "my_role_user" / "user-1" val body = """{"name":"Jhon","age":12,"hobby":["coding"]}""" - Then("calling without authentication returns 401") - val resp401 = makePostRequest(callUrl.POST, body) - resp401.code should equal(401) - resp401.body.extract[ErrorMessage].message should include(AuthenticatedUserIsRequired) - - Then("calling authenticated but without the role returns 403") - val resp403 = makePostRequest(callUrl.POST <@ (user1), body) - resp403.code should equal(403) - resp403.body.extract[ErrorMessage].message should include(UserHasMissingRoles) - - Then("granting the role makes the call succeed (200)") - Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, dynamicRole) - val resp200 = makePostRequest(callUrl.POST <@ (user1), body) - resp200.code should equal(200) - json.compactRender(resp200.body) should include("_from_path") + assertRoleGated401Then403Then200(callUrl, body, dynamicRole) { resp200 => + json.compactRender(resp200.body) should include("_from_path") + } } // Regression guard for DynamicEndpointCodeGenerator.buildTemplate: the template served by diff --git a/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala b/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala index ba69e99330..e33eee4eb5 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala @@ -21,9 +21,10 @@ import code.metadata.comments.MappedComment import code.metadata.narrative.MappedNarrative import code.metadata.transactionimages.MappedTransactionImage import code.metadata.wheretags.MappedWhereTag +import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, UserHasMissingRoles} import code.setup.{APIResponse, DefaultUsers, ServerSetupWithTestData} import code.transactionattribute.MappedTransactionAttribute -import com.openbankproject.commons.model.{AccountId, AccountRoutingJsonV121, AmountOfMoneyJsonV121, BankId, CreateViewJson, UpdateViewJSON} +import com.openbankproject.commons.model.{AccountId, AccountRoutingJsonV121, AmountOfMoneyJsonV121, BankId, CreateViewJson, ErrorMessage, UpdateViewJSON} import com.openbankproject.commons.util.ApiShortVersions import code.setup.OBPReq import org.json4s.native.Serialization.write @@ -42,6 +43,30 @@ trait V400ServerSetup extends ServerSetupWithTestData with DefaultUsers { def dynamicEndpoint_Request: OBPReq = baseRequest / "obp" / ApiShortVersions.`dynamic-endpoint`.toString def dynamicEntity_Request: OBPReq = baseRequest / "obp" / ApiShortVersions.`dynamic-entity`.toString + /** + * Shared by DynamicResourceDocTest and DynamicResourceDocJavaTest: exercises + * ResourceDoc.authCheckIO's role-gated path against a runtime-compiled dynamic-resource-doc -- + * calling without auth returns 401, authenticated without the role returns 403, and granting the + * role makes the call succeed (200), handed to `assertSuccess` for endpoint-specific checks. + */ + def assertRoleGated401Then403Then200(callUrl: OBPReq, body: String, dynamicRole: String)(assertSuccess: APIResponse => Unit): Unit = { + Then("calling without authentication returns 401") + val resp401 = makePostRequest(callUrl.POST, body) + resp401.code should equal(401) + resp401.body.extract[ErrorMessage].message should include(AuthenticatedUserIsRequired) + + Then("calling authenticated but without the role returns 403") + val resp403 = makePostRequest(callUrl.POST <@ (user1), body) + resp403.code should equal(403) + resp403.body.extract[ErrorMessage].message should include(UserHasMissingRoles) + + Then("granting the role makes the call succeed (200)") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, dynamicRole) + val resp200 = makePostRequest(callUrl.POST <@ (user1), body) + resp200.code should equal(200) + assertSuccess(resp200) + } + def randomBankId : String = { def getBanksInfo : APIResponse = { val request = v4_0_0_Request / "banks" From 3ec0b993279248b3c6780a4063b6ae65697c8b60 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 17:26:48 +0200 Subject: [PATCH 37/59] test: enable the Berlin Group v1.3 alias by default so it gets HTTP-level coverage berlin_group_v1_3_alias_path could not be toggled per-test at runtime: its ScannedApiVersion identity is captured once by ScannedApis. versionMapScannedApis' process-wide classpath scan (a lazy val, shared across the whole JVM/shard), which gets forced by the first unrelated request that falls through Http4sApp's route chain -- almost always long before any test-specific setPropsValues call. The only way to exercise a real alias operation id end to end is to have the prop already set before the JVM boots. Set berlin_group_v1_3_alias_path=0.6/v1 in test.default.props (local) and both CI workflows' generated test.default.props (build_pull_request. yml, build_container.yml). Add a regression test in ApiCollectionEndpointTest mirroring the existing per-standard coverage (OBPv6.0.0/UK Open Banking/Berlin Group v1.3 canonical) for the alias's BGv1-getPaymentInitiationStatus operation id, and pin the same operation id in ResourceDocRegistryParityTest alongside the existing BGv2-getAccountDetails pin. --- .github/workflows/build_container.yml | 3 +++ .github/workflows/build_pull_request.yml | 3 +++ .../util/ResourceDocRegistryParityTest.scala | 7 ++++++ .../v4_0_0/ApiCollectionEndpointTest.scala | 25 ++++++++++++++++++- 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_container.yml b/.github/workflows/build_container.yml index f9f87b6762..9893d4d4e6 100644 --- a/.github/workflows/build_container.yml +++ b/.github/workflows/build_container.yml @@ -289,6 +289,9 @@ jobs: echo ResetPasswordUrlEnabled=true >> obp-api/src/main/resources/props/test.default.props echo consents.allowed=true >> obp-api/src/main/resources/props/test.default.props echo hikari.maximumPoolSize=20 >> obp-api/src/main/resources/props/test.default.props + # Enables the Berlin Group v1.3 alias so ResourceDocRegistryParityTest and + # ApiCollectionEndpointTest can exercise a real alias operation id end to end. + echo berlin_group_v1_3_alias_path=0.6/v1 >> obp-api/src/main/resources/props/test.default.props echo write_metrics=false >> obp-api/src/main/resources/props/test.default.props # Log emails instead of opening a real SMTP socket: without this, # LocalMappedConnector.sendCustomerNotification's EMAIL branch calls diff --git a/.github/workflows/build_pull_request.yml b/.github/workflows/build_pull_request.yml index 39716fe115..c18b9ae0d8 100644 --- a/.github/workflows/build_pull_request.yml +++ b/.github/workflows/build_pull_request.yml @@ -283,6 +283,9 @@ jobs: echo ResetPasswordUrlEnabled=true >> obp-api/src/main/resources/props/test.default.props echo consents.allowed=true >> obp-api/src/main/resources/props/test.default.props echo hikari.maximumPoolSize=20 >> obp-api/src/main/resources/props/test.default.props + # Enables the Berlin Group v1.3 alias so ResourceDocRegistryParityTest and + # ApiCollectionEndpointTest can exercise a real alias operation id end to end. + echo berlin_group_v1_3_alias_path=0.6/v1 >> obp-api/src/main/resources/props/test.default.props echo write_metrics=false >> obp-api/src/main/resources/props/test.default.props # Log emails instead of opening a real SMTP socket: without this, # LocalMappedConnector.sendCustomerNotification's EMAIL branch calls diff --git a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala index 341c65e122..9daddf9850 100644 --- a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala +++ b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala @@ -57,5 +57,12 @@ class ResourceDocRegistryParityTest extends ServerSetup { scenario("the operation id from the sandbox bug report resolves", RegistryParityTag) { allOperationIds should contain("BGv2-getAccountDetails") } + + // berlin_group_v1_3_alias_path is set in test.default.props precisely so this surface is + // reliably non-empty here (see that file's comment on why it must be set before boot, not + // toggled per-test) -- pin a concrete alias operation id, not just the generic loop above. + scenario("the operation id from the Berlin Group v1.3 alias resolves", RegistryParityTag) { + allOperationIds should contain("BGv1-getPaymentInitiationStatus") + } } } diff --git a/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala index 755f1f1869..492d6524ff 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala @@ -224,6 +224,29 @@ class ApiCollectionEndpointTest extends V400ServerSetup { val operationId= apiCollectionEndpoint.operation_id } + { + // Regression pin for the Berlin Group v1.3 alias gap: berlin_group_v1_3_alias_path + // (set to 0.6/v1 in test.default.props) activates Http4sBGv13Alias, whose docs are + // re-stamped copies of the canonical BG v1.3 docs carrying their own operation ids + // (BGv1-... here, not BGv1.3-...). These were served by the resource-docs dispatcher + // via ScannedApis discovery but missing from the global operation-id union, the same + // class of gap as BGv2 above. + Then(s"we test the $ApiEndpoint6- BGv1-getPaymentInitiationStatus (Berlin Group v1.3 alias)") + val requestApiCollectionEndpoint = (v4_0_0_Request / "my" / "api-collection-ids" / apiCollectionId / "api-collection-endpoints").POST <@ (user1) + + lazy val postApiCollectionEndpointJson = SwaggerDefinitionsJSON.postApiCollectionEndpointJson400.copy(operation_id="BGv1-getPaymentInitiationStatus") + + val responseApiCollectionEndpointJson = makePostRequest(requestApiCollectionEndpoint, write(postApiCollectionEndpointJson)) + Then("We should get a 201") + responseApiCollectionEndpointJson.code should equal(201) + val apiCollectionEndpoint = responseApiCollectionEndpointJson.body.extract[ApiCollectionEndpointJson400] + + apiCollectionEndpoint.operation_id should be (postApiCollectionEndpointJson.operation_id) + apiCollectionEndpoint.api_collection_endpoint_id shouldNot be (null) + + val operationId= apiCollectionEndpoint.operation_id + } + { Then(s"we test the $ApiEndpoint7") val requestGet = (v4_0_0_Request / "my" / "api-collection-ids" / apiCollectionId / "api-collection-endpoints").GET <@ (user1) @@ -234,7 +257,7 @@ class ApiCollectionEndpointTest extends V400ServerSetup { val apiCollectionsJsonGet400 = responseGet.body.extract[ApiCollectionEndpointsJson400] - apiCollectionsJsonGet400.api_collection_endpoints.length should be (5) + apiCollectionsJsonGet400.api_collection_endpoints.length should be (6) } } } From 2a754c7456c67baef8ce871a6cdefecc5add7eef Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 17:35:38 +0200 Subject: [PATCH 38/59] test: regenerate frozen_type_meta_data.txt to match the updated blob Kept in sync with frozen_type_meta_data (regenerated by the previous commit) via code.util.FrozenMetaDataText -- FrozenMetaDataTextTest fails when the two disagree. --- obp-api/src/test/resources/frozen_type_meta_data.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/obp-api/src/test/resources/frozen_type_meta_data.txt b/obp-api/src/test/resources/frozen_type_meta_data.txt index d7bddd02ce..bf2252ef5b 100644 --- a/obp-api/src/test/resources/frozen_type_meta_data.txt +++ b/obp-api/src/test/resources/frozen_type_meta_data.txt @@ -511,7 +511,6 @@ field code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON.SeverJWK use String field code.api.util.APIUtil.BooleanBody value Boolean field code.api.util.APIUtil.EndpointInfo name String field code.api.util.APIUtil.EndpointInfo version String -field code.api.util.APIUtil.JArrayBody value org.json4s.JArray field code.api.v1_2_1.APIInfoJSON connector String field code.api.v1_2_1.APIInfoJSON git_commit String field code.api.v1_2_1.APIInfoJSON hosted_by code.api.v1_2_1.HostedBy @@ -3145,6 +3144,7 @@ field code.dynamicResourceDoc.JsonDynamicResourceDoc errorResponseBodies String field code.dynamicResourceDoc.JsonDynamicResourceDoc exampleRequestBody Option[org.json4s.JValue] field code.dynamicResourceDoc.JsonDynamicResourceDoc methodBody String field code.dynamicResourceDoc.JsonDynamicResourceDoc partialFunctionName String +field code.dynamicResourceDoc.JsonDynamicResourceDoc programmingLang String field code.dynamicResourceDoc.JsonDynamicResourceDoc requestUrl String field code.dynamicResourceDoc.JsonDynamicResourceDoc requestVerb String field code.dynamicResourceDoc.JsonDynamicResourceDoc roles String From ff17b5e038001307bd0f77471ef3520a27c6e5b2 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 17:58:25 +0200 Subject: [PATCH 39/59] test: pin a v7-only operation id as resolvable, covering the third drift instance The global operation-id union used to be built from the v6.0.0 aggregation, so operation ids belonging to endpoints that exist only in v7.0.0 were absent from it and could not be added to an API collection. That drift instance had no regression test: the OBPv6.0.0-* cases in ApiCollectionEndpointTest pass under both the old v6-based union and the current v7-based one, so they cannot detect it. Pin OBPv7.0.0-getMyMetrics (v7-only -- not part of Http4sResourceDocAggregation.v600) as a real api-collection-endpoint request, and add the matching named pin in ResourceDocRegistryParityTest alongside the BGv2 and Berlin Group v1.3 alias ones, so all three historical drift instances now have both HTTP-level and registry-level coverage. --- .../util/ResourceDocRegistryParityTest.scala | 9 +++++++ .../v4_0_0/ApiCollectionEndpointTest.scala | 25 ++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala index 9daddf9850..401ab2b241 100644 --- a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala +++ b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala @@ -54,6 +54,9 @@ class ResourceDocRegistryParityTest extends ServerSetup { } } + // The three named pins below are the three historical drift instances. The generic loop + // above would also catch them, but naming them keeps the specific regressions legible. + scenario("the operation id from the sandbox bug report resolves", RegistryParityTag) { allOperationIds should contain("BGv2-getAccountDetails") } @@ -64,5 +67,11 @@ class ResourceDocRegistryParityTest extends ServerSetup { scenario("the operation id from the Berlin Group v1.3 alias resolves", RegistryParityTag) { allOperationIds should contain("BGv1-getPaymentInitiationStatus") } + + // The union used to be built from the v6.0.0 aggregation, so v7-only operation ids were + // absent from it. getMyMetrics exists only in v7.0.0, so it pins the v7 base specifically. + scenario("a v7-only operation id resolves", RegistryParityTag) { + allOperationIds should contain("OBPv7.0.0-getMyMetrics") + } } } diff --git a/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala index 492d6524ff..9f1ce7e465 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala @@ -247,6 +247,29 @@ class ApiCollectionEndpointTest extends V400ServerSetup { val operationId= apiCollectionEndpoint.operation_id } + { + // Regression pin for the third drift instance: the global operation-id union used to be + // built from the v6.0.0 aggregation, so operation ids belonging to endpoints that exist + // ONLY in v7.0.0 (getMyMetrics, getTopUsers, getTopConsumers) were absent from it and + // could not be added to an API collection either. getMyMetrics is v7-only -- it is not + // part of Http4sResourceDocAggregation.v600 -- so this pins the v7 base specifically, + // unlike the OBPv6.0.0-* cases above which passed even under the old v6-based union. + Then(s"we test the $ApiEndpoint6- OBPv7.0.0-getMyMetrics (v7-only endpoint)") + val requestApiCollectionEndpoint = (v4_0_0_Request / "my" / "api-collection-ids" / apiCollectionId / "api-collection-endpoints").POST <@ (user1) + + lazy val postApiCollectionEndpointJson = SwaggerDefinitionsJSON.postApiCollectionEndpointJson400.copy(operation_id="OBPv7.0.0-getMyMetrics") + + val responseApiCollectionEndpointJson = makePostRequest(requestApiCollectionEndpoint, write(postApiCollectionEndpointJson)) + Then("We should get a 201") + responseApiCollectionEndpointJson.code should equal(201) + val apiCollectionEndpoint = responseApiCollectionEndpointJson.body.extract[ApiCollectionEndpointJson400] + + apiCollectionEndpoint.operation_id should be (postApiCollectionEndpointJson.operation_id) + apiCollectionEndpoint.api_collection_endpoint_id shouldNot be (null) + + val operationId= apiCollectionEndpoint.operation_id + } + { Then(s"we test the $ApiEndpoint7") val requestGet = (v4_0_0_Request / "my" / "api-collection-ids" / apiCollectionId / "api-collection-endpoints").GET <@ (user1) @@ -257,7 +280,7 @@ class ApiCollectionEndpointTest extends V400ServerSetup { val apiCollectionsJsonGet400 = responseGet.body.extract[ApiCollectionEndpointsJson400] - apiCollectionsJsonGet400.api_collection_endpoints.length should be (6) + apiCollectionsJsonGet400.api_collection_endpoints.length should be (7) } } } From 3a50efdc55b8269cb4f3f8b62f91f4809c191f4d Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 18:15:26 +0200 Subject: [PATCH 40/59] fix: make Java dynamic-code dependency validation actually enforce Two bugs made createJavaHttp4sEndpoint's validation a no-op in practice, discovered by manually enabling dynamic_code_compile_validate_enable against a real Java method_body that calls a non-whitelisted OBP method: - Javassist's LoaderClassPath reads a class's bytecode via classLoader.getResourceAsStream(...), but the compiler's ch.obermuhlner.scriptengine.java.MemoryClassLoader only overrides loadClass() and never exposes compiled bytes as a classpath resource. getDynamicCodeDependentMethods silently found nothing to check no matter what the Java code called. Fix: read the bytes from the classloader's private byte map and register them with Javassist directly via ByteArrayClassPath. - Once dependency lookup worked, a rejection was still swallowed: JsonResponseException never sets a Throwable message, so wrapping the validation call in Box.tryo turned it into Failure(null, Full(theException), Empty), and DynamicEndpoints.scala's `case Failure(msg: String, ...)` pattern fails to match a null msg -- silently falling through to a generic "compiled code return nothing" error and discarding the real rejection reason. Fix: run validation outside any Box.tryo, mirroring how the Scala path's CompiledObjects.validateDependency() (also never tryo-wrapped) already lets the exception propagate uncaught. Manually verified end-to-end: a Java method_body calling code.api.util.APIUtil.getPropsValue (not on the dependency whitelist) is now rejected with 400 OBP-40046, naming the exact forbidden call. --- .../scala/code/api/util/DynamicUtil.scala | 39 +++++++++++++++++-- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 6cbb891aed..98392f7dd4 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -624,7 +624,11 @@ object DynamicUtil extends MdcLoggable{ import scala.jdk.CollectionConverters._ - Box tryo { + // Real compile happens here (javax.tools.JavaCompiler via the JSR-223 "java" engine) — any + // Java syntax/type error surfaces as an exception, caught by this `Box tryo` and turned into + // a Failure. Deliberately narrow: only the compile step is wrapped. Validation runs below, + // outside any tryo — see the comment there for why that separation matters. + val compiledScriptBox: Box[ch.obermuhlner.scriptengine.java.JavaCompiledScript] = Box tryo { val packageExp = UUID.randomUUID().toString.replaceAll("^|-", "_") val packageMatcher = Pattern.compile("""(?m)^\s*package\s+\S+?\s*;""").matcher(methodBody) @@ -632,10 +636,37 @@ object DynamicUtil extends MdcLoggable{ |${packageMatcher.replaceFirst("")} |""".stripMargin - // Real compile happens here (javax.tools.JavaCompiler via the JSR-223 "java" engine) — - // any Java syntax/type error surfaces as an exception, caught by the outer `Box tryo`. - val compiledScript = javaEngine.asInstanceOf[javax.script.Compilable].compile(javaCode) + javaEngine.asInstanceOf[javax.script.Compilable].compile(javaCode) .asInstanceOf[ch.obermuhlner.scriptengine.java.JavaCompiledScript] + } + + // Deliberately outside compiledScriptBox's `Box tryo`: a rejection here throws + // JsonResponseException, which must propagate UNCAUGHT (mirroring the Scala path's + // CompiledObjects.validateDependency(), also never wrapped in tryo) so + // compileDynamicResourceDoc's `case e: JsonResponseException => throw e` sees it intact. + // JsonResponseException never sets a Throwable message (getMessage == null); Box.tryo would + // catch it into Failure(null, Full(theException), Empty), and DynamicEndpoints.scala's + // `case Failure(msg: String, ...)` pattern silently fails to match a null msg — falling + // through to "compiled code return nothing" and discarding the real rejection reason. `.map` + // does not swallow exceptions the way `Box tryo` does, so this stays uncaught here. + compiledScriptBox.map { compiledScript => + val compiledClass = compiledScript.getCompiledClass + + // getDynamicCodeDependentMethods loads the class's bytecode via Javassist's LoaderClassPath, + // which reads it through classLoader.getResourceAsStream(...). The compiler's + // ch.obermuhlner.scriptengine.java.MemoryClassLoader only overrides loadClass() — it never + // exposes the compiled bytes as a classpath resource — so that lookup silently fails + // (javassist.NotFoundException) and validation would see zero dependent methods no matter + // what the Java code actually calls. Read the bytes directly from the classloader's private + // byte map (reflection is unavoidable here: java-scriptengine exposes no public accessor) + // and hand them to Javassist explicitly via ByteArrayClassPath, so the real method bodies — + // including any restricted OBP call — are visible to validation. + val classBytesField = compiledClass.getClassLoader.getClass.getDeclaredField("mapClassBytes") + classBytesField.setAccessible(true) + val classBytes = classBytesField.get(compiledClass.getClassLoader) + .asInstanceOf[java.util.Map[String, Array[Byte]]].get(compiledClass.getName) + getClassPool(compiledClass.getClassLoader) + .appendClassPath(new javassist.ByteArrayClassPath(compiledClass.getName, classBytes)) // Validate the real compiled Supplier class before it's ever invoked — see the doc comment // above for why this must run against getCompiledInstance, not `func`/`this.partialFunction`. From d252dc5f4b1189f780d724af95a752a1760c6ac0 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 22:13:19 +0200 Subject: [PATCH 41/59] fix: build the global union from the current OBP surface only, in a defined order Two defects from the registry refactor, both in how allStaticResourceDocs was assembled. Folding every per-version aggregation into the union added 287 operation ids it never carried (the older aggregations are not subsets of the v7 one -- an endpoint dropped after v4 keeps its operation id there), and 234 of those collide on partialFunctionName with an entry already present. Http4s600's top-apis and popular-apis and JSONFactory6.0.0's metrics all build `partialFunctionName -> operationId` with `.toMap`, where the last entry wins, so with v1.2.1 sorting last the reported operation_id flipped to the oldest id: getBanks became OBPv1.2.1-getBanks, root became OBPv1.2.1-root. Restrict the union to obpUnionVersion (the current OBP aggregation) plus every non-OBP standard. Consequence, deliberate and documented at the constant: an operation id living only in a superseded aggregation stays unresolvable, exactly as before the refactor. The scanned half of the registry was a plain Map, so the same `.toMap` consumers resolved a partialFunctionName shared by two scanned standards according to hash iteration order -- undefined, and free to shift when a standard is added or removed. The Berlin Group v1.3 alias re-stamps the canonical BG v1.3 docs and so collides with BG v2 on getAccountDetails and four other names, and test.default.props now activates that alias for every test run. Sort it by fullyQualifiedVersion into a ListMap; BG v2 then wins those names, matching the behaviour before this branch. ResourceDocRegistryParityTest follows the narrowed union and regains the per-surface non-empty assertion, without which a standard whose docs stop being registered passes as a trivial subset. A new scenario pins obpUnionVersion as the newest OBP-standard version in the registry, so adding a v8 aggregation without moving it fails instead of silently dropping v8-only operation ids. Verified against a running instance: OBPv1.2.1-getBanks and OBPv3.0.0-getAggregateMetrics are rejected with OBP-40048 again, while BGv2-getAccountDetails, BGv1-getPaymentInitiationStatus and OBPv7.0.0-getMyMetrics still resolve. Full local suite 3573/0. --- .../code/api/util/ResourceDocRegistry.scala | 64 +++++++++++++++---- .../util/ResourceDocRegistryParityTest.scala | 49 ++++++++++---- 2 files changed, 89 insertions(+), 24 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala b/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala index f0194d0aca..4b308f3e15 100644 --- a/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala +++ b/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala @@ -3,7 +3,7 @@ package code.api.util import code.api.berlin.group.ConstantsBG import code.api.util.APIUtil.ResourceDoc import com.openbankproject.commons.util.ApiVersion._ -import com.openbankproject.commons.util.ApiVersion +import com.openbankproject.commons.util.{ApiStandards, ApiVersion, ScannedApiVersion} import scala.collection.immutable.ListMap @@ -61,22 +61,62 @@ object ResourceDocRegistry { // Every standard discovered via ScannedApis (UK OB 200/310/401, BG v1.3 canonical + alias, // BG v2, and any future `with ScannedApis` standard). Explicit entries win on key collision -- // BG v1.3 canonical is both explicit above and a registrant, same underlying buffer either way. - val scanned: Map[ApiVersion, () => Seq[ResourceDoc]] = ScannedApis.versionMapScannedApis.collect { - case (version, apis) if !explicit.contains(version) => version -> (() => apis.allResourceDocs.toSeq) - } + // + // Sorted, and folded into a ListMap, so the registry has ONE defined iteration order. + // ScannedApis.versionMapScannedApis is an unordered Map, and several standards share + // partialFunctionNames (the BG v1.3 alias re-stamps the canonical BG v1.3 docs, so it collides + // with both BG v1.3 and -- on getAccountDetails, getAccountList, getCardAccountBalances, + // getCardAccountTransactionList, getTransactionDetails -- with BG v2). Consumers such as + // Http4s600's top-apis/popular-apis and JSONFactory6.0.0's metrics build + // `partialFunctionName -> operationId` with `.toMap`, where the LAST entry wins, so leaving the + // order to Map's hash iteration would leave the reported operation_id undefined and let it + // shift silently whenever a standard is added or removed. + val scanned: ListMap[ApiVersion, () => Seq[ResourceDoc]] = + ScannedApis.versionMapScannedApis.toSeq + .collect { case (version: ScannedApiVersion, apis) if !explicit.contains(version) => + version -> (() => apis.allResourceDocs.toSeq) } + .sortBy(_._1.fullyQualifiedVersion) + .foldLeft(ListMap.empty[ApiVersion, () => Seq[ResourceDoc]])(_ + _) explicit ++ scanned } /** What the per-version resource-docs dispatcher serves for this version (empty if unknown). */ def docsFor(version: ApiVersion): Seq[ResourceDoc] = registry.get(version).map(_ ()).getOrElse(Nil) - /** The global operation-id union. Excludes the dynamic arms: those are runtime-mutable (created/ - * deleted dynamic entities and endpoints) and are appended FRESH by APIUtil.getAllResourceDocs on - * every call -- caching them in this lazy union would serve stale dynamic docs. Deduped by - * operationId: the underlying per-version buffers legitimately overlap (each OBP-standard - * aggregation repeats every older version's docs), and consumers only ever `.find`/build a - * lookup map from this list, never rely on it containing duplicates. */ + /** + * The OBP-standard surface the global union is built from. + * + * The registry also holds the cumulative aggregations for every older OBP version, because the + * dispatcher must still serve /resource-docs/OBPv4.0.0/obp and friends. Those are NOT folded into + * the union: they are not subsets of the v7 aggregation (an endpoint dropped after v4 keeps its + * operation id there), so including them would add ~287 operation ids that the union never + * carried, 234 of which collide on partialFunctionName with an entry already present -- and the + * `.toMap` consumers above would then report the OLDEST id (getBanks -> OBPv1.2.1-getBanks) + * instead of the current one in metrics, top-apis and popular-apis output. + * + * Consequence, deliberately accepted: an operation id that exists ONLY in a superseded + * aggregation stays unresolvable by api-collection-endpoint creation, exactly as before this + * refactor. ResourceDocRegistryParityTest pins that this constant is the newest OBP-standard + * version in the registry, so adding v8 without moving it fails the build rather than silently + * dropping v8-only operation ids from the union. + */ + val obpUnionVersion: ApiVersion = v7_0_0 + + private def isObpStandard(version: ApiVersion): Boolean = version match { + case sv: ScannedApiVersion => sv.apiStandard == ApiStandards.obp.toString + case _ => false + } + + /** Versions whose docs make up the global union: the current OBP surface plus every non-OBP + * standard. Excluding the other OBP-standard keys also excludes `dynamic-endpoint` / + * `dynamic-entity` (both carry apiStandard "obp"), which must stay out for a second reason: + * they are runtime-mutable and APIUtil.getAllResourceDocs appends them FRESH on every call, so + * caching them in this lazy union would serve stale dynamic docs. */ + lazy val unionVersions: Seq[ApiVersion] = + registry.keys.filter(v => v == obpUnionVersion || !isObpStandard(v)).toSeq + + /** The global operation-id union. Deduped by operationId: the surfaces legitimately overlap, and + * consumers only ever `.find`/build a lookup map from this list, never rely on duplicates. */ lazy val allStaticResourceDocs: List[ResourceDoc] = - (registry - `dynamic-endpoint` - `dynamic-entity`) - .values.flatMap(_ ()).toList.distinctBy(_.operationId) + unionVersions.flatMap(docsFor).toList.distinctBy(_.operationId) } diff --git a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala index 401ab2b241..3e3167c83a 100644 --- a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala +++ b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala @@ -1,7 +1,7 @@ package code.api.util import code.setup.ServerSetup -import com.openbankproject.commons.util.{ApiVersion, ScannedApiVersion} +import com.openbankproject.commons.util.{ApiStandards, ApiVersion, ScannedApiVersion} import org.scalatest.Tag /** @@ -30,23 +30,28 @@ class ResourceDocRegistryParityTest extends ServerSetup { case other => other.toString } - // Dynamic arms (dynamic-endpoint / dynamic-entity) are excluded: they are runtime-mutable and - // ResourceDocRegistry.allStaticResourceDocs itself excludes them for the same reason (see its - // doc comment) -- APIUtil.getAllResourceDocs appends them fresh via allDynamicResourceDocs - // instead, so comparing them against this registry-derived snapshot would be meaningless. + // Scoped to ResourceDocRegistry.unionVersions -- the current OBP surface plus every non-OBP + // standard. The superseded OBP aggregations (v6.0.0 and older) and the two dynamic arms are + // deliberately out of the union; see ResourceDocRegistry.obpUnionVersion for why, and for the + // accepted consequence that an operation id living only in a superseded aggregation stays + // unresolvable. private lazy val surfaces: List[(String, Seq[String])] = - (ResourceDocRegistry.registry - ApiVersion.`dynamic-endpoint` - ApiVersion.`dynamic-entity`) - .toList - .map { case (version, docsThunk) => (label(version), docsThunk().map(_.operationId)) } + ResourceDocRegistry.unionVersions.toList + .map(version => (label(version), ResourceDocRegistry.docsFor(version).map(_.operationId))) - feature("getAllResourceDocs contains every per-standard resource-doc surface the dispatcher can serve") { + feature("getAllResourceDocs contains every per-standard resource-doc surface the union covers") { scenario("the registry itself is non-empty", RegistryParityTag) { surfaces should not be empty - surfaces.exists(_._2.nonEmpty) shouldBe true } surfaces.foreach { case (label, operationIds) => scenario(s"$label operation ids are all resolvable globally", RegistryParityTag) { + // Non-empty matters as much as membership: an empty surface is trivially a subset of the + // union, so without this a standard whose docs silently stop being registered (the very + // failure mode this test exists for) would pass unnoticed. + withClue(s"$label contributed no operation ids at all -- did its docs stop being registered? ") { + operationIds should not be empty + } val missing = operationIds.filterNot(allOperationIds.contains) withClue(s"$label operation ids missing from getAllResourceDocs: ${missing.take(10).mkString(", ")} ") { missing shouldBe empty @@ -54,8 +59,28 @@ class ResourceDocRegistryParityTest extends ServerSetup { } } - // The three named pins below are the three historical drift instances. The generic loop - // above would also catch them, but naming them keeps the specific regressions legible. + // Guards the one hand-maintained knob left in the registry: if a v8.0.0 aggregation is added + // without moving obpUnionVersion, the union would keep serving the v7 surface and every + // v8-only operation id would silently be unresolvable -- the exact bug this PR started from. + scenario("obpUnionVersion is the newest OBP-standard version in the registry", RegistryParityTag) { + val obpVersions = ResourceDocRegistry.registry.keys.toList.collect { + case sv: ScannedApiVersion + if sv.apiStandard == ApiStandards.obp.toString && + sv != ApiVersion.`dynamic-endpoint` && sv != ApiVersion.`dynamic-entity` => sv + } + obpVersions should not be empty + // ApiVersionUtils.versions lists the OBP versions oldest-first, so the highest index wins. + val newest = obpVersions.maxBy(ApiVersionUtils.versions.indexOf(_)) + withClue(s"registry holds OBP versions ${obpVersions.map(_.fullyQualifiedVersion).mkString(", ")} " + + s"but obpUnionVersion is ${ResourceDocRegistry.obpUnionVersion} ") { + newest shouldBe ResourceDocRegistry.obpUnionVersion + } + } + + // The three named pins below are the three historical drift instances. They are NOT redundant + // with the loop above: both sides of that loop are now derived from ResourceDocRegistry, so its + // membership half holds by construction and cannot fail. What the loop still catches is a + // surface going empty; what these pins still catch is a specific operation id disappearing. scenario("the operation id from the sandbox bug report resolves", RegistryParityTag) { allOperationIds should contain("BGv2-getAccountDetails") From 6ce26699a3a441e110a8ecd859ffcce5c4487a4c Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 23:04:39 +0200 Subject: [PATCH 42/59] test: cover Java dynamic-code dependency validation end-to-end DynamicUtil.Validation's dependency whitelist and Constant.SHOW_USED_CONNECTOR_METHODS were final vals, computed once when first touched (in practice during server boot, before any test scenario runs). setPropsValues -- the standard per-scenario props override used throughout this test suite -- mutates Props.lockedProviders, which only reaches code that re-reads Props on each call; it cannot un-freeze an already-computed field. That made this security path untestable without a real server restart, which is how the earlier manual verification (see prior commit) had to prove it. Changed the whole SHOW_USED_CONNECTOR_METHODS / Validation dependency chain from val to def so it re-evaluates from live props on each call. This costs nothing extra in production: the one expensive step, DynamicUtil.compileScalaCodeUnchecked, is already memoized by the exact source string, so re-evaluating is a cache hit unless the underlying props value actually changed. DynamicResourceDocJavaSecurityValidationTest exercises the resulting create-time HTTP path directly: a Java method_body calling code.api.util.APIUtil.getPropsValue (not on the dependency whitelist) is rejected with 400 OBP-40046, naming the exact forbidden call -- the same assertion the manual verification made by hand. --- .../scala/code/api/constant/constant.scala | 7 +- .../scala/code/api/util/DynamicUtil.scala | 25 ++++--- ...esourceDocJavaSecurityValidationTest.scala | 70 +++++++++++++++++++ 3 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala diff --git a/obp-api/src/main/scala/code/api/constant/constant.scala b/obp-api/src/main/scala/code/api/constant/constant.scala index d4bfed683b..fa51de2af9 100644 --- a/obp-api/src/main/scala/code/api/constant/constant.scala +++ b/obp-api/src/main/scala/code/api/constant/constant.scala @@ -305,7 +305,12 @@ object Constant extends MdcLoggable { final val CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL: Int = APIUtil.getPropsValue(s"createLocalisedResourceDocJson.cache.ttl.seconds", "3600").toInt final val GET_DYNAMIC_RESOURCE_DOCS_TTL: Int = APIUtil.getPropsValue(s"dynamicResourceDocsObp.cache.ttl.seconds", "3600").toInt final val GET_STATIC_RESOURCE_DOCS_TTL: Int = APIUtil.getPropsValue(s"staticResourceDocsObp.cache.ttl.seconds", "3600").toInt - final val SHOW_USED_CONNECTOR_METHODS: Boolean = APIUtil.getPropsAsBoolValue(s"show_used_connector_methods", false) + // def, not final val: DynamicUtil.Validation.validateDependency (dynamic-code dependency + // checking) needs this to react to a props change without a restart -- e.g. test-time + // setPropsValues overrides. A final val here would freeze at whatever value was true the + // moment this object was first touched (typically during server boot, well before any test + // scenario runs), and no later prop override could ever reach it. + def SHOW_USED_CONNECTOR_METHODS: Boolean = APIUtil.getPropsAsBoolValue(s"show_used_connector_methods", false) // Rate Limiting Cache Prefixes (with global namespace and versioning) // Both call_counter and rl_active are versioned for consistent cache invalidation diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 98392f7dd4..42a1537263 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -361,10 +361,17 @@ object DynamicUtil extends MdcLoggable{ object Validation { - val dynamicCodeSandboxPermissions = APIUtil.getPropsValue("dynamic_code_sandbox_permissions", "[]").trim - val scalaCodePermissioins = "List[java.security.Permission]"+dynamicCodeSandboxPermissions.replaceFirst("\\[","(").dropRight(1)+")" - val permissions:Box[List[java.security.Permission]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodePermissioins) - + // def, not val, throughout this object: these must react to a props change (e.g. test-time + // setPropsValues) without a restart, not freeze at whatever the props held the moment + // Validation was first touched (typically by whichever dynamic-code test happens to run + // first in a shared test JVM). This costs nothing extra in production -- the only expensive + // step, DynamicUtil.compileScalaCodeUnchecked, is already memoized by the exact source + // string, so re-evaluating these on every call is a cache hit unless the underlying props + // value actually changed. + def dynamicCodeSandboxPermissions = APIUtil.getPropsValue("dynamic_code_sandbox_permissions", "[]").trim + def scalaCodePermissioins = "List[java.security.Permission]"+dynamicCodeSandboxPermissions.replaceFirst("\\[","(").dropRight(1)+")" + def permissions:Box[List[java.security.Permission]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodePermissioins) + // all Permissions put at here // Here is the Java Permission document, please extend these permissions carefully. // https://docs.oracle.com/javase/8/docs/technotes/guides/security/spec/security-spec.doc3.html#17001 @@ -383,15 +390,15 @@ object DynamicUtil extends MdcLoggable{ // new RuntimePermission("accessDeclaredMembers"), // new RuntimePermission("getClassLoader"), // ) - val allowedRuntimePermissions = permissions.openOrThrowException("Can not compile the props `dynamic_code_sandbox_permissions` to permissions") + def allowedRuntimePermissions = permissions.openOrThrowException("Can not compile the props `dynamic_code_sandbox_permissions` to permissions") - val dependenciesString = APIUtil.getPropsValue("dynamic_code_compile_validate_dependencies", "[]").trim + def dependenciesString = APIUtil.getPropsValue("dynamic_code_compile_validate_dependencies", "[]").trim // `Map[String, String](` rather than `Map(`: the props default is an empty list, and a bare // `Map()` leaves its type parameters undetermined, so the trailing .toMap cannot prove the // elements are pairs and the reflective compilation fails. The .toMap itself is needed because // mapValues returns a view rather than a Map. - val scalaCodeDependencies = s"${DynamicUtil.importStatements}"+dependenciesString.replaceFirst("\\[","Map[String, String](").dropRight(1) +").mapValues(v => StringUtils.split(v, ',').map(_.trim).toSet).toMap" - val dependenciesBox: Box[Map[String, Set[String]]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodeDependencies) + def scalaCodeDependencies = s"${DynamicUtil.importStatements}"+dependenciesString.replaceFirst("\\[","Map[String, String](").dropRight(1) +").mapValues(v => StringUtils.split(v, ',').map(_.trim).toSet).toMap" + def dependenciesBox: Box[Map[String, Set[String]]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodeDependencies) /** * Compilation OBP Dependencies Guard, only checked the OBP methods, not scala/Java libraies(are checked during the runtime.). @@ -422,7 +429,7 @@ object DynamicUtil extends MdcLoggable{ // PractiseEndpoint.getClass.getTypeName + "*" -> "*", // // ).mapValues(v => StringUtils.split(v, ',').map(_.trim).toSet) - val allowedCompilationMethods: Map[String, Set[String]] = dependenciesBox.openOrThrowException("Can not compile the props `dynamic_code_compile_validate_dependencies` to Map") + def allowedCompilationMethods: Map[String, Set[String]] = dependenciesBox.openOrThrowException("Can not compile the props `dynamic_code_compile_validate_dependencies` to Map") //Do not touch this Set, try to use the `allowedPermissions` and `allowedMethods` to control the sandbox val restrictedTypes = Set( diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala new file mode 100644 index 0000000000..e54d7a5f19 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala @@ -0,0 +1,70 @@ +package code.api.v4_0_0 + +import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.util.ApiRole +import code.api.util.ErrorMessages.DynamicResourceDocMethodDependency +import code.entitlement.Entitlement +import com.openbankproject.commons.model.ErrorMessage +import org.json4s.native.Serialization.write + +/** + * With dynamic_code_compile_validate_enable=true, a Java method_body that calls an OBP method NOT + * on the dependency whitelist must be rejected -- proving createJavaHttp4sEndpoint validates the + * real compiled Java class (getCompiledInstance), not just its Scala wrapper. + */ +class DynamicResourceDocJavaSecurityValidationTest extends V400ServerSetup { + + private def maliciousMethodBody: String = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaSecurityProbe implements Supplier> { + | private Object apply(Object[] args) { + | // APIUtil.getPropsValue is NOT on dynamic_code_compile_validate_dependencies' + | // whitelist (only errorJsonResponse*/scalaFutureToLaFuture/futureToBoxedResponse are). + | String secret = code.api.util.APIUtil$.MODULE$.getPropsValue("hostname", "none"); + | Map response = new LinkedHashMap<>(); + | response.put("leaked", secret); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + feature("Security validation of Java method_body against dynamic_code_compile_validate_dependencies") { + scenario("Registering a Java doc that calls a non-whitelisted OBP method is rejected with 400") { + setPropsValues( + "show_used_connector_methods" -> "true", + "dynamic_code_compile_validate_enable" -> "true", + "dynamic_code_compile_validate_dependencies" -> + """[NewStyle.function.getClass.getTypeName -> "*", CompiledObjects.getClass.getTypeName -> "sandbox", HttpCode.getClass.getTypeName -> "200", DynamicCompileEndpoint.getClass.getTypeName -> "getPathParams, scalaFutureToBoxedJsonResponse", APIUtil.getClass.getTypeName -> "errorJsonResponse, errorJsonResponse$default$1, errorJsonResponse$default$2, errorJsonResponse$default$3, errorJsonResponse$default$4, scalaFutureToLaFuture, futureToBoxedResponse", ErrorMessages.getClass.getTypeName -> "*", ExecutionContext.Implicits.getClass.getTypeName -> "global", JSONFactory400.getClass.getTypeName -> "createBanksJson", classOf[Sandbox].getTypeName -> "runInSandbox", classOf[CallContext].getTypeName -> "*", classOf[ResourceDoc].getTypeName -> "getPathParams", "scala.reflect.runtime.package$" -> "universe", PractiseEndpoint.getClass.getTypeName + "*" -> "*"]""" + ) + + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "securityProbeTest", + requestUrl = "/security_probe_test/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(maliciousMethodBody, "UTF-8"), + programmingLang = "Java" + ) + val resp = makePostRequest(createReq, write(doc)) + + Then("the compile is rejected with 400 DynamicResourceDocMethodDependency, not accepted") + resp.code should equal(400) + resp.body.extract[ErrorMessage].message should include(DynamicResourceDocMethodDependency) + } + } +} From 39c131ab468fabeebc56e65bd8be305ad770698a Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 23:52:23 +0200 Subject: [PATCH 43/59] fix: treat an unset berlin_group_v1_3_alias_path as no alias at all "".split("/") returns Array(""), not an empty array, so berlinGroupV13AliasPath was List("") on a default instance -- nonEmpty. Every downstream `if (berlinGroupV13AliasPath.nonEmpty)` guard therefore took its ACTIVE branch with an empty prefix: Http4sBGv13Alias published 55 docs stamped with the degenerate ScannedApiVersion("", "", ""), whose operation ids came out as `BG-`, and its route bridge matched the prefix "/" (every request) only to fall through again. That was invisible while the alias sat outside the global operation-id union. Now that this branch folds it in, those 55 junk ids became resolvable: verified against a running default instance that api-collection-endpoint creation accepted BG-getAccountDetails and BG-getPaymentInitiationStatus with 201, naming endpoints no route serves. Filtering empty segments makes "unset" mean "inactive" again -- both now return 400, while BGv1.3, BGv2, UK and OBP ids are unaffected and /resource-docs/BGv1.3/obp still serves its 55 docs. OBP_BERLIN_GROUP_1_3_Alias.apiVersion has to guard .head/.last against the now genuinely empty list: the ScannedApis classpath scan catches a throwing companion and only logs a warning, so an unguarded NoSuchElementException would drop the alias silently. Inactive registrations keep the empty-string version, which deliberately does not equal ConstantsBG.berlinGroupVersion1 -- colliding there would let this doc-less object win ScannedApis' .toMap and blank out the canonical BG v1.3 resource docs. The alias assertions in both tests no longer depend on a prop that only exists in a gitignored file. test.default.props is excluded by .gitignore:21, so the CI workflows carried berlin_group_v1_3_alias_path while a fresh clone or an IDE runner did not: deleting the line locally reproduced two failures whose messages gave no hint a prop was missing. They now cancel with an explanatory message when the alias is inactive, and read the expected operation id back from the alias's own docs instead of hard-coding the BGv1- prefix, which is derived from the configured path. Verified both ways: with the prop set 13/13 pass, without it 11 pass and 2 cancel. Full local suite 3573/0. --- .../v1_3/OBP_BERLIN_GROUP_1_3_Alias.scala | 16 ++++++++- .../main/scala/code/api/util/APIUtil.scala | 9 ++++- .../util/ResourceDocRegistryParityTest.scala | 35 +++++++++++++++---- .../v4_0_0/ApiCollectionEndpointTest.scala | 33 ++++++++++------- 4 files changed, 72 insertions(+), 21 deletions(-) diff --git a/obp-api/src/main/scala/code/api/berlin/group/v1_3/OBP_BERLIN_GROUP_1_3_Alias.scala b/obp-api/src/main/scala/code/api/berlin/group/v1_3/OBP_BERLIN_GROUP_1_3_Alias.scala index 2beba04807..8fc872c200 100644 --- a/obp-api/src/main/scala/code/api/berlin/group/v1_3/OBP_BERLIN_GROUP_1_3_Alias.scala +++ b/obp-api/src/main/scala/code/api/berlin/group/v1_3/OBP_BERLIN_GROUP_1_3_Alias.scala @@ -47,8 +47,22 @@ import scala.collection.mutable.ArrayBuffer */ object OBP_BERLIN_GROUP_1_3_Alias extends OBPRestHelper with MdcLoggable with ScannedApis { + /** + * The version this aggregator registers under. + * + * `berlinGroupV13AliasPath` is empty when `berlin_group_v1_3_alias_path` is unset, so `.head` / + * `.last` must be guarded: this object is instantiated by the ScannedApis classpath scan, which + * catches a throwing companion and merely logs a warning, so an unguarded NoSuchElementException + * would drop the alias silently. Inactive registrations keep the empty-string version they have + * always had, which no request can address and which deliberately does NOT equal + * ConstantsBG.berlinGroupVersion1 -- colliding with the canonical BG v1.3 key would let this + * (doc-less) object win ScannedApis' `.toMap` and blank out /resource-docs/BGv1.3/obp. + */ override val apiVersion: ScannedApiVersion = - ScannedApiVersion(berlinGroupV13AliasPath.head, berlinGroupV13AliasPath.head, berlinGroupV13AliasPath.last) + if (berlinGroupV13AliasPath.nonEmpty) + ScannedApiVersion(berlinGroupV13AliasPath.head, berlinGroupV13AliasPath.head, berlinGroupV13AliasPath.last) + else + ScannedApiVersion("", "", "") val versionStatus: String = ApiVersionStatus.DRAFT.toString diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index 4d9f61bcff..b9b110654c 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -4756,7 +4756,14 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ } ) - val berlinGroupV13AliasPath = APIUtil.getPropsValue("berlin_group_v1_3_alias_path","").split("/").toList.map(_.trim) + // Empty segments are dropped so that "unset" really means "no alias". Without the filter an + // absent prop yields List("") -- "".split("/") returns Array(""), not an empty array -- which is + // nonEmpty, so every `if (berlinGroupV13AliasPath.nonEmpty)` guard downstream took its ACTIVE + // branch on a default instance: Http4sBGv13Alias published 55 docs stamped with the degenerate + // version ScannedApiVersion("", "", ""), whose operation ids came out as `BG-`, and its + // route bridge matched on the prefix "/" (every path) only to fall through again. + val berlinGroupV13AliasPath = + APIUtil.getPropsValue("berlin_group_v1_3_alias_path","").split("/").toList.map(_.trim).filter(_.nonEmpty) val getAtmsIsPublic = APIUtil.getPropsAsBoolValue("apiOptions.getAtmsIsPublic", true) diff --git a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala index 3e3167c83a..f574d1a42b 100644 --- a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala +++ b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala @@ -1,5 +1,6 @@ package code.api.util +import code.api.berlin.group.v1_3.{Http4sBGv13Alias, OBP_BERLIN_GROUP_1_3_Alias} import code.setup.ServerSetup import com.openbankproject.commons.util.{ApiStandards, ApiVersion, ScannedApiVersion} import org.scalatest.Tag @@ -25,7 +26,19 @@ class ResourceDocRegistryParityTest extends ServerSetup { private lazy val allOperationIds: Set[String] = APIUtil.getAllResourceDocs.map(_.operationId).toSet + // The Berlin Group v1.3 alias is the one surface in the union a deployment can switch off: + // berlin_group_v1_3_alias_path is unset by default and is supplied for test runs by + // test.default.props and by the two CI workflows. test.default.props is gitignored + // (.gitignore:21), so a fresh clone, a colleague's checkout or an IDE ScalaTest run may not have + // it -- assertions that genuinely need the alias cancel there instead of failing with a message + // that gives no hint a prop is missing. + private lazy val aliasVersion: ApiVersion = OBP_BERLIN_GROUP_1_3_Alias.apiVersion + private lazy val aliasIsConfigured: Boolean = Http4sBGv13Alias.resourceDocs.nonEmpty + private val aliasNotConfigured = + "berlin_group_v1_3_alias_path is not set, so the Berlin Group v1.3 alias contributes no docs" + private def label(version: ApiVersion): String = version match { + case v if v == aliasVersion && !aliasIsConfigured => "Berlin Group v1.3 alias (not configured)" case sv: ScannedApiVersion => sv.fullyQualifiedVersion case other => other.toString } @@ -35,17 +48,18 @@ class ResourceDocRegistryParityTest extends ServerSetup { // deliberately out of the union; see ResourceDocRegistry.obpUnionVersion for why, and for the // accepted consequence that an operation id living only in a superseded aggregation stays // unresolvable. - private lazy val surfaces: List[(String, Seq[String])] = + private lazy val surfaces: List[(ApiVersion, String, Seq[String])] = ResourceDocRegistry.unionVersions.toList - .map(version => (label(version), ResourceDocRegistry.docsFor(version).map(_.operationId))) + .map(version => (version, label(version), ResourceDocRegistry.docsFor(version).map(_.operationId))) feature("getAllResourceDocs contains every per-standard resource-doc surface the union covers") { scenario("the registry itself is non-empty", RegistryParityTag) { surfaces should not be empty } - surfaces.foreach { case (label, operationIds) => + surfaces.foreach { case (version, label, operationIds) => scenario(s"$label operation ids are all resolvable globally", RegistryParityTag) { + if (version == aliasVersion && !aliasIsConfigured) cancel(aliasNotConfigured) // Non-empty matters as much as membership: an empty surface is trivially a subset of the // union, so without this a standard whose docs silently stop being registered (the very // failure mode this test exists for) would pass unnoticed. @@ -86,11 +100,18 @@ class ResourceDocRegistryParityTest extends ServerSetup { allOperationIds should contain("BGv2-getAccountDetails") } - // berlin_group_v1_3_alias_path is set in test.default.props precisely so this surface is - // reliably non-empty here (see that file's comment on why it must be set before boot, not - // toggled per-test) -- pin a concrete alias operation id, not just the generic loop above. + // The alias's operation-id prefix is derived from the configured path (0.6/v1 in the test props + // yields BGv1-...), so the expected id is read back from the alias's own docs rather than + // hard-coded -- a deployment that configures a different path would otherwise fail here for no + // real reason. scenario("the operation id from the Berlin Group v1.3 alias resolves", RegistryParityTag) { - allOperationIds should contain("BGv1-getPaymentInitiationStatus") + if (!aliasIsConfigured) cancel(aliasNotConfigured) + val aliasOperationId = Http4sBGv13Alias.resourceDocs + .find(_.partialFunctionName == "getPaymentInitiationStatus").map(_.operationId) + withClue("the alias is configured but publishes no getPaymentInitiationStatus doc ") { + aliasOperationId shouldBe defined + } + allOperationIds should contain(aliasOperationId.get) } // The union used to be built from the v6.0.0 aggregation, so v7-only operation ids were diff --git a/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala index 9f1ce7e465..26d40aa469 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala @@ -27,6 +27,7 @@ package code.api.v4_0_0 import org.json4s._ import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.berlin.group.v1_3.Http4sBGv13Alias import code.api.util.APIUtil.OAuth._ import code.api.v4_0_0.APIMethods400.Implementations4_0_0 import com.github.dwickern.macros.NameOf.nameOf @@ -224,17 +225,24 @@ class ApiCollectionEndpointTest extends V400ServerSetup { val operationId= apiCollectionEndpoint.operation_id } - { - // Regression pin for the Berlin Group v1.3 alias gap: berlin_group_v1_3_alias_path - // (set to 0.6/v1 in test.default.props) activates Http4sBGv13Alias, whose docs are - // re-stamped copies of the canonical BG v1.3 docs carrying their own operation ids - // (BGv1-... here, not BGv1.3-...). These were served by the resource-docs dispatcher - // via ScannedApis discovery but missing from the global operation-id union, the same - // class of gap as BGv2 above. - Then(s"we test the $ApiEndpoint6- BGv1-getPaymentInitiationStatus (Berlin Group v1.3 alias)") + // Regression pin for the Berlin Group v1.3 alias gap: when berlin_group_v1_3_alias_path is + // set (0.6/v1 in test.default.props and in both CI workflows) Http4sBGv13Alias publishes + // re-stamped copies of the canonical BG v1.3 docs under their own operation ids -- served by + // the resource-docs dispatcher via ScannedApis discovery, but formerly missing from the + // global operation-id union, the same class of gap as BGv2 above. + // + // Guarded on the alias actually being configured, and the expected id is read back from its + // own docs rather than hard-coded: test.default.props is gitignored (.gitignore:21), so a + // fresh clone or an IDE runner may not carry that prop, and a deployment may configure a + // different path (which changes the id's prefix). + val aliasOperationId: Option[String] = Http4sBGv13Alias.resourceDocs + .find(_.partialFunctionName == "getPaymentInitiationStatus").map(_.operationId) + + aliasOperationId.foreach { opId => + Then(s"we test the $ApiEndpoint6- $opId (Berlin Group v1.3 alias)") val requestApiCollectionEndpoint = (v4_0_0_Request / "my" / "api-collection-ids" / apiCollectionId / "api-collection-endpoints").POST <@ (user1) - lazy val postApiCollectionEndpointJson = SwaggerDefinitionsJSON.postApiCollectionEndpointJson400.copy(operation_id="BGv1-getPaymentInitiationStatus") + lazy val postApiCollectionEndpointJson = SwaggerDefinitionsJSON.postApiCollectionEndpointJson400.copy(operation_id = opId) val responseApiCollectionEndpointJson = makePostRequest(requestApiCollectionEndpoint, write(postApiCollectionEndpointJson)) Then("We should get a 201") @@ -243,8 +251,6 @@ class ApiCollectionEndpointTest extends V400ServerSetup { apiCollectionEndpoint.operation_id should be (postApiCollectionEndpointJson.operation_id) apiCollectionEndpoint.api_collection_endpoint_id shouldNot be (null) - - val operationId= apiCollectionEndpoint.operation_id } { @@ -280,7 +286,10 @@ class ApiCollectionEndpointTest extends V400ServerSetup { val apiCollectionsJsonGet400 = responseGet.body.extract[ApiCollectionEndpointsJson400] - apiCollectionsJsonGet400.api_collection_endpoints.length should be (7) + // Six unconditional cases above, plus the Berlin Group v1.3 alias one when that alias is + // configured for this run. + val expected = if (aliasOperationId.isDefined) 7 else 6 + apiCollectionsJsonGet400.api_collection_endpoints.length should be (expected) } } } From 3125596ae2846b18d298f03fc47c9c541b0b877c Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 00:08:26 +0200 Subject: [PATCH 44/59] fix: give the registry a tie-free order and a guard that cannot be bypassed Two follow-ups from reviewing the registry work itself. The scanned half was sorted by fullyQualifiedVersion, which concatenates apiStandard.toUpperCase and apiShortVersion and can therefore collide across distinct keys -- ("BG", "v1.3") and ("BGV", "1.3") both render "BGV1.3", and berlin_group_v1_3_alias_path lets a deployment choose the alias's half of such a pair. sortBy is only stable with respect to its input, and the input is the unordered ScannedApis.versionMapScannedApis, so a tie would hand the order back to hash iteration and with it the `.toMap` winner for a shared partialFunctionName. Sort by (apiStandard, apiShortVersion) instead: that pair is exactly ScannedApiVersion's equals/hashCode key, so two distinct keys of that Map always differ in it and the order is total. The resulting sequence is unchanged -- alias, BG v1.3, BG v2, UK 2.0/3.1/4.0.1 -- so BG v2 keeps winning the names it shares with the alias. The obpUnionVersion guard ranked versions with ApiVersionUtils.versions.indexOf, which returns -1 for anything absent from that equally hand-maintained list. A -1 loses every maxBy comparison, so adding a v8.0.0 aggregation to the registry while forgetting ApiVersionUtils.versions left v7 as the maximum and the scenario green -- precisely the two-places-to-edit slip it was written to catch. Assert first that every OBP version in the registry can be ranked at all. Verified by injecting an unregistered OBPv8.0.0: the guard now fails with "OBP versions in the registry but missing from ApiVersionUtils.versions: OBPv8.0.0", where before it passed. Full local suite 3573/0. --- .../scala/code/api/util/ResourceDocRegistry.scala | 12 +++++++++++- .../api/util/ResourceDocRegistryParityTest.scala | 14 +++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala b/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala index 4b308f3e15..882c4683e3 100644 --- a/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala +++ b/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala @@ -71,11 +71,21 @@ object ResourceDocRegistry { // `partialFunctionName -> operationId` with `.toMap`, where the LAST entry wins, so leaving the // order to Map's hash iteration would leave the reported operation_id undefined and let it // shift silently whenever a standard is added or removed. + // + // The sort key is (apiStandard, apiShortVersion) rather than fullyQualifiedVersion because it + // cannot tie: that pair is exactly ScannedApiVersion's equals/hashCode key, so two distinct + // keys of this Map always differ in it, and sortBy therefore yields a total order rather than + // falling back to the unordered input for ties. fullyQualifiedVersion concatenates the two + // (apiStandard.toUpperCase + apiShortVersion) and so can collide across distinct keys -- + // ("BG", "v1.3") and ("BGV", "1.3") both render "BGV1.3" -- which a deployment could reach by + // configuring berlin_group_v1_3_alias_path. The resulting order is unchanged in practice: + // alias ("0.6"/"") first, then BG v1.3, BG v2, then UK 2.0/3.1/4.0.1, so BG v2 keeps winning + // the names it shares with the alias, matching the behaviour before this branch. val scanned: ListMap[ApiVersion, () => Seq[ResourceDoc]] = ScannedApis.versionMapScannedApis.toSeq .collect { case (version: ScannedApiVersion, apis) if !explicit.contains(version) => version -> (() => apis.allResourceDocs.toSeq) } - .sortBy(_._1.fullyQualifiedVersion) + .sortBy(entry => (entry._1.apiStandard, entry._1.apiShortVersion)) .foldLeft(ListMap.empty[ApiVersion, () => Seq[ResourceDoc]])(_ + _) explicit ++ scanned } diff --git a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala index f574d1a42b..25b065255b 100644 --- a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala +++ b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala @@ -83,7 +83,19 @@ class ResourceDocRegistryParityTest extends ServerSetup { sv != ApiVersion.`dynamic-endpoint` && sv != ApiVersion.`dynamic-entity` => sv } obpVersions should not be empty - // ApiVersionUtils.versions lists the OBP versions oldest-first, so the highest index wins. + + // Rank by position in ApiVersionUtils.versions, which lists the OBP versions oldest-first. + // indexOf returns -1 for anything absent from that (also hand-maintained) list, and a -1 + // would lose every maxBy comparison -- so a v8.0.0 added to the registry but not to + // ApiVersionUtils.versions would leave v7 as the maximum and let this scenario pass, in + // exactly the two-places-to-edit case it exists to catch. Establish coverage first. + val unranked = obpVersions.filter(ApiVersionUtils.versions.indexOf(_) < 0) + withClue(s"OBP versions in the registry but missing from ApiVersionUtils.versions: " + + s"${unranked.map(_.fullyQualifiedVersion).mkString(", ")} -- add them there so they can be " + + s"ranked, otherwise this guard cannot see them ") { + unranked shouldBe empty + } + val newest = obpVersions.maxBy(ApiVersionUtils.versions.indexOf(_)) withClue(s"registry holds OBP versions ${obpVersions.map(_.fullyQualifiedVersion).mkString(", ")} " + s"but obpUnionVersion is ${ResourceDocRegistry.obpUnionVersion} ") { From 0318885ab7b6721ae841eb97dfe063fe56babdf7 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Tue, 1 Sep 2026 08:29:46 +0200 Subject: [PATCH 45/59] =?UTF-8?q?Added=20auth=5Ftype=20metrics=20column=20?= =?UTF-8?q?any=20grant=20targeting=20a=20consent=20user=20redirects=20to?= =?UTF-8?q?=20its=20granting=20human=20with=20a=20warn-log;=20sole=20exemp?= =?UTF-8?q?tion=20is=20createdByProcess=20=3D=3D=20consent=5Fuser.=20Conse?= =?UTF-8?q?ntUtil.addEntitlements=20tags=20its=20writes=20accordingly.=20?= =?UTF-8?q?=20=20-=20process=20column=20retired:=20parameter=20and=20acces?= =?UTF-8?q?sor=20removed=20from=20the=20=20=20=20=20Entilement=20trait=20a?= =?UTF-8?q?nd=20implementation=20=20=20=20=20group=20queries=20now=20key?= =?UTF-8?q?=20on=20group=5Fid;=20GroupEntitlementJsonV600=20exposes=20=20?= =?UTF-8?q?=20=20=20created=5Fby=5Fprocess=20instead=20of=20process.=20=20?= =?UTF-8?q?=20-=20Explicit-target=20guards=20(400,=20"=E2=80=A6names=20a?= =?UTF-8?q?=20consent=20user=E2=80=A6"):=20=20=20=20=20addEntitlement=20v2?= =?UTF-8?q?.0.0=20+=20v7.0.0;=20addUserToGroup=20v6;=20=20=20=20=20grantUs?= =?UTF-8?q?erAccessToViewById=20v5.1;=20createAccountAccessRequest=20v6=20?= =?UTF-8?q?=20=20=20=20(reject=20at=20creation)=20plus=20repeated=20check?= =?UTF-8?q?=20at=20its=20approval;=20=20=20=20=20createAccount=20endpoints?= =?UTF-8?q?=20v2.0.0,=20v3.1.0,=20v4.0.0=20(regular=20+=20=20=20=20=20sett?= =?UTF-8?q?lement),=20v5.0.0,=20v7.0.0.=20=20=20-=20Implicit-target=20reso?= =?UTF-8?q?lution=20to=20the=20accountable=20user=20(currently=20a=20=20?= =?UTF-8?q?=20=20=20human):=20bank-creator=20grants=20(v2.2,=20v5,=20v6,?= =?UTF-8?q?=20v7=20incl.=20the=20=20=20=20=20generated-bank=20endpoint),?= =?UTF-8?q?=20v6=20=20=20=20=20dynamic-entity=20creator=20roles,=20v3.0.0?= =?UTF-8?q?=20entitlement-request=20requester,=20=20=20=20=20createAccount?= =?UTF-8?q?=20owner=20fallbacks,=20and=20the=20connector-internal=20=20=20?= =?UTF-8?q?=20=20HOLDING-account=20holder.=20=20=20-=20Rename:=20effective?= =?UTF-8?q?HumanUserId=20=E2=86=92=20accountableUserId?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../SwaggerDefinitionsJSON.scala | 1 + .../scala/code/api/constant/constant.scala | 15 ++++ .../main/scala/code/api/util/ApiSession.scala | 25 +++++-- .../scala/code/api/util/ConsentUtil.scala | 7 +- .../scala/code/api/util/WriteMetricUtil.scala | 33 ++++++++- .../code/api/util/migration/Migration.scala | 13 ++++ .../MigrationOfActivityDashboardIndexes.scala | 4 +- .../migration/MigrationOfMetricAuthType.scala | 74 +++++++++++++++++++ .../scala/code/api/v2_0_0/Http4s200.scala | 16 +++- .../scala/code/api/v2_2_0/Http4s220.scala | 9 ++- .../scala/code/api/v3_0_0/Http4s300.scala | 8 +- .../scala/code/api/v3_1_0/Http4s310.scala | 8 +- .../scala/code/api/v4_0_0/Http4s400.scala | 16 +++- .../scala/code/api/v5_0_0/Http4s500.scala | 17 ++++- .../scala/code/api/v5_1_0/Http4s510.scala | 6 ++ .../scala/code/api/v6_0_0/Http4s600.scala | 57 ++++++++++---- .../code/api/v6_0_0/JSONFactory6.0.0.scala | 9 ++- .../scala/code/api/v7_0_0/Http4s700.scala | 32 +++++--- .../LocalMappedConnectorInternal.scala | 8 +- .../scala/code/entitlement/Entilement.scala | 4 +- .../code/entitlement/MappedEntitlements.scala | 52 +++++++++---- .../main/scala/code/metrics/APIMetrics.scala | 11 ++- .../code/metrics/ElasticsearchMetrics.scala | 6 +- .../scala/code/metrics/MappedMetrics.scala | 28 ++++++- .../code/metrics/MetricBatchWriter.scala | 12 +-- .../code/model/dataAccess/ResourceUser.scala | 2 +- .../scheduler/MetricsArchiveScheduler.scala | 3 +- .../code/api/util/AgentDelegationTest.scala | 14 ++-- .../code/api/v6_0_0/MetricAuthTypeTest.scala | 60 +++++++++++++++ .../test/scala/code/metrics/MetricsTest.scala | 18 ++--- 30 files changed, 464 insertions(+), 104 deletions(-) create mode 100644 obp-api/src/main/scala/code/api/util/migration/MigrationOfMetricAuthType.scala create mode 100644 obp-api/src/test/scala/code/api/v6_0_0/MetricAuthTypeTest.scala diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala index 2ca0eeef91..a3c8cac50a 100644 --- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala +++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala @@ -3202,6 +3202,7 @@ object SwaggerDefinitionsJSON { operation_id = "OBPv4.0.0-getBanks", api_instance_id = "obp_node_a", consent_reference_id = Some(ExampleValue.consentReferenceIdExample.value), + auth_type = Some("Consent"), certificate_trust = Some("forwarded"), certificate_trust_detail = Some("cn=nginx-prod-1,ou=edge,o=tesobe gmbh,c=de") ) diff --git a/obp-api/src/main/scala/code/api/constant/constant.scala b/obp-api/src/main/scala/code/api/constant/constant.scala index d4bfed683b..2cb79f2af5 100644 --- a/obp-api/src/main/scala/code/api/constant/constant.scala +++ b/obp-api/src/main/scala/code/api/constant/constant.scala @@ -16,6 +16,21 @@ object Constant extends MdcLoggable { final val directLoginHeaderName = "DirectLogin" + // createdByProcess of entitlement rows the consent engine copies onto a consent user — + // the per-consent principal a Consent-JWT authenticates as (its ResourceUser row carries + // CreatedByConsentId). Only rows tagged with this value may target a consent user: + // addEntitlement redirects any other grant to the consent's granting human, so durable + // roles (e.g. bank-creator grants) can never strand on a principal that dies with its + // consent. Also the marker for cleaning these rows up when the consent is revoked. + final val consent_user = "consent_user" + + // createdByProcess of entitlement rows granted through group membership (the Groups + // feature). The value predates this constant: the Groups feature originally wrote it to + // its own `process` column, a duplicate of createdByProcess since retired — provenance + // now lives in createdByProcess like every other granting mechanism, and group rows are + // identified by their group_id. + final val group_membership = "GROUP_MEMBERSHIP" + object Pagination { final val offset = 0 final val limit = 50 diff --git a/obp-api/src/main/scala/code/api/util/ApiSession.scala b/obp-api/src/main/scala/code/api/util/ApiSession.scala index cdd812050d..fd125ef95e 100644 --- a/obp-api/src/main/scala/code/api/util/ApiSession.scala +++ b/obp-api/src/main/scala/code/api/util/ApiSession.scala @@ -39,7 +39,7 @@ case class CallContext( // the creator is the granting human (they create their own consent in the Portal). // Not set by Berlin Group / UK flows, where the consent may be created by a TPP flow // with no human logged in — see `consenter` for those. - // Read via humanUser / effectiveHumanUserId, where it takes precedence over consenter. + // Read via humanUser / accountableUserId, where it takes precedence over consenter. onBehalfOfUser: Box[User] = Empty, // The human (PSU) who AUTHORISED the consent this request runs under — the owner of // record, from the consent table's userId (bound by updateConsentUser during the @@ -113,7 +113,7 @@ case class CallContext( * Anything that must name a human rather than a principal reads this instead: the CBS adapter, * which tells the core banking system who is asking, and the consent ownership checks. * Stored data (metric rows included) always carries the authenticated principal; the human is - * resolved at read time via the consent table (see effectiveHumanUserId). + * resolved at read time via the consent table (see accountableUserId). */ def humanUser: Box[User] = onBehalfOfUser.or(consenter).or(user) @@ -182,7 +182,7 @@ case class CallContext( // (CallContext.user), never a resolved human. Under a consent that principal is the // consent's own shadow user (a per-consent UUID with an empty name) — the on-behalf-of // human is not stored here but resolved at read time via the consent table - // (consentReferenceId below -> consent.userId), see CallContext.effectiveHumanUserId. + // (consentReferenceId below -> consent.userId), see CallContext.accountableUserId. userId = this.user.map(_.userId).toOption, userName = this.user.map(_.name).toOption, consumerId = this.consumer.map(_.consumerId.get).toOption, @@ -217,22 +217,31 @@ case class CallContext( def userId: String = user.map(_.userId).openOrThrowException(AuthenticatedUserIsRequired) /** - * The human User this request is really about. + * The ACCOUNTABLE identity this request is really about — the user_id that durable + * state (creator role grants, account holders, entitlement requests) and attribution + * (metrics families, "my" queries) bind to. "Accountable" deliberately hints at a + * legal person: today resolution always ends at the human who granted the consent, + * but the contract is accountability, not species — if durable, sponsored agent + * identities are ever admitted as principals in their own right, resolution may stop + * at such an agent without this name becoming a lie (unlike the previous name, + * effectiveHumanUserId). * - * The authenticated `user` may be the human themselves, or an agent user minted by a - * Consent the human granted (e.g. Opey / MCP acting under a consent). Resolution order: + * The authenticated `user` may be the accountable party themselves, or a consent user + * minted by a Consent they granted (e.g. Opey / MCP acting under a consent) — consent + * users are ephemeral and must never hold durable state (see addEntitlement's guard). + * Resolution order: * 1. `onBehalfOfUser` or `consenter`, when a middleware populated them (free); * 2. otherwise resolve via the delegation registry: the caller's ResourceUser row's * CreatedByConsentId names the Consent that minted it, and that Consent's userId * names the granting human; - * 3. otherwise the caller IS the human. + * 3. otherwise the caller IS the accountable party. * * IMPORTANT: this reads only the authenticated user and server-written columns * (ResourceUser.CreatedByConsentId, MappedConsent.mUserId). It deliberately takes no * parameters so nothing caller-asserted (body/header/query values) can ever influence * the resolution — identity-sensitive queries (e.g. /my/banks) depend on that. */ - def effectiveHumanUserId: String = { + def accountableUserId: String = { val delegatedHumanUserId = onBehalfOfUser.or(consenter).map(_.userId).filter(_.nonEmpty) delegatedHumanUserId.openOr { val authenticatedUserId = user.map(_.userId).openOr("") diff --git a/obp-api/src/main/scala/code/api/util/ConsentUtil.scala b/obp-api/src/main/scala/code/api/util/ConsentUtil.scala index f55cd0cfb3..f17d281d67 100644 --- a/obp-api/src/main/scala/code/api/util/ConsentUtil.scala +++ b/obp-api/src/main/scala/code/api/util/ConsentUtil.scala @@ -380,7 +380,10 @@ object Consent extends MdcLoggable { existingEntitlements.exists(_.roleName == entitlement.role_name) match { // Check is a role already added to a user case false => val bankId = if (role.requiresBankId) entitlement.bank_id else "" - Entitlement.entitlement.vend.addEntitlement(bankId, user.userId, entitlement.role_name) match { + // Tagged consent_user: this is the ONE writer allowed to target a consent + // user — addEntitlement redirects untagged grants to the granting human. + Entitlement.entitlement.vend.addEntitlement(bankId, user.userId, entitlement.role_name, + createdByProcess = Constant.consent_user) match { case Full(_) => (entitlement, "AddedOrExisted") case _ => (entitlement, CannotAddEntitlement + entitlement) @@ -905,7 +908,7 @@ object Consent extends MdcLoggable { } yield { (principal, callContext.copy( // The PSU stays reachable for everything that needs a human: the CBS adapter, metric - // attribution, and CallContext.effectiveHumanUserId. + // attribution, and CallContext.accountableUserId. consenter = Full(psu), ukConsentId = Some(storedConsent.consentId), consentReferenceId = Some(storedConsent.consentReferenceId) diff --git a/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala b/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala index 8e7768e92a..5af1eb65ae 100644 --- a/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala +++ b/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala @@ -45,7 +45,8 @@ object WriteMetricUtil extends MdcLoggable { duration: Long, responseBodyToWrite: String, sourceIp: String, - targetIp: String) + targetIp: String, + authType: String) private def persistAndPublishMetric(responseBody: Any, cc: CallContextLight): Unit = { val fields = MetricFields( @@ -58,7 +59,8 @@ object WriteMetricUtil extends MdcLoggable { duration = callDuration(cc), responseBodyToWrite = responseBodyForMetric(responseBody, cc), sourceIp = requestHeaderValue(cc, "x-forwarded-for"), - targetIp = requestHeaderValue(cc, "x-forwarded-host") + targetIp = requestHeaderValue(cc, "x-forwarded-host"), + authType = deriveAuthType(cc) ) // enqueue synchronously so flush() in tests reliably drains this metric before assertions @@ -74,6 +76,30 @@ object WriteMetricUtil extends MdcLoggable { } } + /** + * Authentication SCHEME of the call — never the credential itself. "Consent" wins + * outright: when a consent authenticated the call, the Authorization header (if any) + * was not what authorized it. The rest is read off the Authorization header shape, + * with the gateway payload / direct-login params as fallbacks for flows that + * populate those without a header. + */ + private[util] def deriveAuthType(cc: CallContextLight): String = { + if (cc.consentReferenceId.isDefined) "Consent" + else cc.authReqHeaderField.map(_.trim) match { + case Some(h) if h.startsWith("DirectLogin") => "DirectLogin" + case Some(h) if h.startsWith("Bearer") => "OAuth2" + case Some(h) if h.startsWith("GatewayLogin") => "GatewayLogin" + case Some(h) if h.startsWith("DAuth") => "DAuth" + case Some(h) if h.startsWith("OAuth") => "OAuth1" + case Some(_) => "Other" + case None => + if (cc.gatewayLoginRequestPayload.isDefined) "GatewayLogin" + else if (cc.directLoginToken != null && cc.directLoginToken.nonEmpty) "DirectLogin" + else if (cc.userId.isDefined) "Other" + else "Anonymous" + } + } + private def callDuration(cc: CallContextLight): Long = (cc.startTime, cc.endTime) match { case (Some(s), Some(e)) => e.getTime - s.getTime @@ -116,7 +142,8 @@ object WriteMetricUtil extends MdcLoggable { code.api.Constant.ApiInstanceId, cc.consentReferenceId.orNull, cc.certificateTrust.orNull, - cc.certificateTrustDetail.orNull + cc.certificateTrustDetail.orNull, + authType ) } catch { case NonFatal(e) => diff --git a/obp-api/src/main/scala/code/api/util/migration/Migration.scala b/obp-api/src/main/scala/code/api/util/migration/Migration.scala index 47eb4c711b..9a094352ef 100644 --- a/obp-api/src/main/scala/code/api/util/migration/Migration.scala +++ b/obp-api/src/main/scala/code/api/util/migration/Migration.scala @@ -157,6 +157,7 @@ object Migration extends MdcLoggable { migrateChatRoomCreatedByAndLastMessageSender() migrateConsentReferenceIdToUuid(startedBeforeSchemifier) migrateMetricConsentReferenceId(startedBeforeSchemifier) + migrateMetricAuthType(startedBeforeSchemifier) migrateMetricCertificateTrust(startedBeforeSchemifier) dropFastFirehoseAccountsViews(startedBeforeSchemifier) alterDynamicResourceDocBodyFieldsLength() @@ -808,6 +809,18 @@ object Migration extends MdcLoggable { } } + private def migrateMetricAuthType(startedBeforeSchemifier: Boolean): Boolean = { + if(startedBeforeSchemifier == true) { + logger.warn(s"Migration.database.migrateMetricAuthType(true) cannot be run before Schemifier.") + true + } else { + val name = nameOf(migrateMetricAuthType(startedBeforeSchemifier)) + runOnce(name) { + MigrationOfMetricAuthType.migrate(name) + } + } + } + private def migrateMetricCertificateTrust(startedBeforeSchemifier: Boolean): Boolean = { if(startedBeforeSchemifier == true) { logger.warn(s"Migration.database.migrateMetricCertificateTrust(true) cannot be run before Schemifier.") diff --git a/obp-api/src/main/scala/code/api/util/migration/MigrationOfActivityDashboardIndexes.scala b/obp-api/src/main/scala/code/api/util/migration/MigrationOfActivityDashboardIndexes.scala index 8716a35c38..278565b0a9 100644 --- a/obp-api/src/main/scala/code/api/util/migration/MigrationOfActivityDashboardIndexes.scala +++ b/obp-api/src/main/scala/code/api/util/migration/MigrationOfActivityDashboardIndexes.scala @@ -88,7 +88,7 @@ object MigrationOfActivityDashboardIndexes { * Index on resourceuser.createdbyconsentid. * * The delegation registry: consent-agent fan-down (/my/metrics, /my/banks) and - * CallContext.effectiveHumanUserId look up agent users by the consent that minted them. + * CallContext.accountableUserId look up agent users by the consent that minted them. * Unindexed this is a full scan of resourceuser on every such request, which matters on * consent-heavy instances where every consent mints a user row. */ @@ -130,7 +130,7 @@ object MigrationOfActivityDashboardIndexes { s"""Added index on resourceuser.createdbyconsentid |Executed SQL: |$executedSql - |Serves the consent-agent delegation fan-down (/my/metrics, /my/banks, effectiveHumanUserId). + |Serves the consent-agent delegation fan-down (/my/metrics, /my/banks, accountableUserId). |""".stripMargin isSuccessful = true saveLog(name, commitId, isSuccessful, startDate, endDate, comment) diff --git a/obp-api/src/main/scala/code/api/util/migration/MigrationOfMetricAuthType.scala b/obp-api/src/main/scala/code/api/util/migration/MigrationOfMetricAuthType.scala new file mode 100644 index 0000000000..c79b85ccfe --- /dev/null +++ b/obp-api/src/main/scala/code/api/util/migration/MigrationOfMetricAuthType.scala @@ -0,0 +1,74 @@ +package code.api.util.migration + +import code.api.util.APIUtil +import code.api.util.migration.Migration.{DbFunction, saveLog} +import code.metrics.MappedMetric +import net.liftweb.mapper.Schemifier + +/** + * Migration: add `auth_type VARCHAR(32)` to both the live `Metric` table and the + * `metricarchive` table — the authentication SCHEME of each call ("Consent", + * "OAuth2", "OAuth1", "DirectLogin", "GatewayLogin", "DAuth", "Anonymous", + * "Other"), never the credential itself. + * + * No backup and no backfill: the column is additive and nullable — historical rows + * legitimately predate it and stay null. No index: always queried alongside the + * indexed date range. + * + * Lift's Schemifier auto-creates the column on fresh deploys from the updated model; + * this migration handles existing deploys. Table name note as in + * MigrationOfMetricConsentReferenceId: unquoted lowercase `metric` everywhere. + */ +object MigrationOfMetricAuthType { + + def migrate(name: String): Boolean = { + DbFunction.tableExists(MappedMetric) match { + case true => + val startDate = System.currentTimeMillis() + val commitId: String = APIUtil.gitCommit + val dbDriver = APIUtil.getPropsValue("db.driver") openOr "org.h2.Driver" + val isMssql = dbDriver.contains("com.microsoft.sqlserver.jdbc.SQLServerDriver") + var isSuccessful = false + val sqlLog = new StringBuilder() + + try { + val addColumnMetric = if (isMssql) { + "ALTER TABLE metric ADD auth_type VARCHAR(32) NULL;" + } else { + "ALTER TABLE metric ADD COLUMN IF NOT EXISTS auth_type VARCHAR(32);" + } + sqlLog.append(DbFunction.maybeWrite(true, Schemifier.infoF _)(() => addColumnMetric)).append("\n") + + val addColumnArchive = if (isMssql) { + "ALTER TABLE metricarchive ADD auth_type VARCHAR(32) NULL;" + } else { + "ALTER TABLE metricarchive ADD COLUMN IF NOT EXISTS auth_type VARCHAR(32);" + } + sqlLog.append(DbFunction.maybeWrite(true, Schemifier.infoF _)(() => addColumnArchive)).append("\n") + + isSuccessful = true + } catch { + case e: Exception => + isSuccessful = false + sqlLog.append(s"\nException: ${e.getMessage}\n") + } + + val endDate = System.currentTimeMillis() + val comment: String = + s"""Executed SQL: + |$sqlLog + |""".stripMargin + saveLog(name, commitId, isSuccessful, startDate, endDate, comment) + isSuccessful + + case false => + val startDate = System.currentTimeMillis() + val commitId: String = APIUtil.gitCommit + val isSuccessful = false + val endDate = System.currentTimeMillis() + val comment: String = s"""${MappedMetric._dbTableNameLC} table does not exist""".stripMargin + saveLog(name, commitId, isSuccessful, startDate, endDate, comment) + isSuccessful + } + } +} diff --git a/obp-api/src/main/scala/code/api/v2_0_0/Http4s200.scala b/obp-api/src/main/scala/code/api/v2_0_0/Http4s200.scala index 38b4915de1..49666dabda 100644 --- a/obp-api/src/main/scala/code/api/v2_0_0/Http4s200.scala +++ b/obp-api/src/main/scala/code/api/v2_0_0/Http4s200.scala @@ -845,8 +845,14 @@ object Http4s200 { isValidID(bank.bankId.value) } loggedInUserId = user.userId - userIdAccountOwner = if (body.user_id.nonEmpty) body.user_id else loggedInUserId + // Implicit owner resolves to the HUMAN: under a Consent the caller is the + // per-consent shadow, and an account held by it strands when the consent dies. + userIdAccountOwner = if (body.user_id.nonEmpty) body.user_id else cc.accountableUserId (postedOrLoggedInUser, cc2) <- NewStyle.function.findByUserId(userIdAccountOwner, Some(cc)) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + _ <- code.util.Helper.booleanToFuture( + s"$InvalidUserId user_id names a consent user (an agent identity minted by a Consent). Accounts are held by humans - use the granting user's USER_ID.", + failCode = 400, cc = cc2)(!postedOrLoggedInUser.isConsentUser) _ <- if (userIdAccountOwner == loggedInUserId) Future.successful(Full(())) else code.util.Helper.booleanToFuture( s"${UserHasMissingRoles} $canCreateAccount or create account for self", failCode = 403, cc = Some(cc)) { @@ -1188,7 +1194,13 @@ object Http4s200 { case req @ POST -> `prefixPath` / "users" / userId / "entitlements" => EndpointHelpers.withUserAndBodyCreated[CreateEntitlementJSON, EntitlementJSON](req) { (user, body, cc) => for { - (_, cc2) <- NewStyle.function.findByUserId(userId, Some(cc)) + (targetUser, cc2) <- NewStyle.function.findByUserId(userId, Some(cc)) + // Explicit target: fail loud rather than redirect. A consent user (an agent + // identity minted by a Consent) cannot hold durable roles — grant to the + // granting human instead. + _ <- code.util.Helper.booleanToFuture( + s"$InvalidUserId USER_ID names a consent user (an agent identity minted by a Consent). Entitlements target humans - use the granting user's USER_ID.", + failCode = 400, cc = cc2)(!targetUser.isConsentUser) role <- Future { unboxFullOrFail( net.liftweb.util.Helpers.tryo { ApiRole.valueOf(body.role_name) }, diff --git a/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala b/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala index f62b53c99a..c619619bb0 100644 --- a/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala +++ b/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala @@ -465,17 +465,20 @@ object Http4s220 { bank.swift_bic, bank.national_identifier, bank.bank_routing.scheme, bank.bank_routing.address, Some(cc) ) + // Creator grants target the HUMAN (see v6.0.0 createBank): under a Consent the + // authenticated user is a per-consent shadow, and roles granted to it are stranded. + humanUserId = cc.accountableUserId entitlements <- Future { unboxFullOrFail( - code.entitlement.Entitlement.entitlement.vend.getEntitlementsByUserId(user.userId), + code.entitlement.Entitlement.entitlement.vend.getEntitlementsByUserId(humanUserId), Some(cc), UnknownError) } _ <- Future { val bankEntitlements = entitlements.filter(_.bankId == bank.id) if (!bankEntitlements.exists(_.roleName == canCreateEntitlementAtOneBank.toString())) - code.entitlement.Entitlement.entitlement.vend.addEntitlement(bank.id, user.userId, canCreateEntitlementAtOneBank.toString()) + code.entitlement.Entitlement.entitlement.vend.addEntitlement(bank.id, humanUserId, canCreateEntitlementAtOneBank.toString(), grantedByUserId = Some(user.userId)) if (!bankEntitlements.exists(_.roleName == canReadDynamicResourceDocsAtOneBank.toString())) - code.entitlement.Entitlement.entitlement.vend.addEntitlement(bank.id, user.userId, canReadDynamicResourceDocsAtOneBank.toString()) + code.entitlement.Entitlement.entitlement.vend.addEntitlement(bank.id, humanUserId, canReadDynamicResourceDocsAtOneBank.toString(), grantedByUserId = Some(user.userId)) } } yield JSONFactory220.createBankJSON(success) } diff --git a/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala b/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala index 4641e6ab44..c89a623173 100644 --- a/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala +++ b/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala @@ -1655,10 +1655,14 @@ object Http4s300 { _ <- code.util.Helper.booleanToFuture( if (ApiRole.valueOf(body.role_name).requiresBankId) EntitlementIsBankRole else EntitlementIsSystemRole, cc = Some(cc)) { ApiRole.valueOf(body.role_name).requiresBankId == body.bank_id.nonEmpty } + // A request for power is a request BY the human: under a Consent the caller is a + // per-consent shadow, and a request filed for it would have an admin granting to + // an identity that dies with the consent (the grant endpoint now rejects that). + requesterUserId = cc.accountableUserId _ <- code.util.Helper.booleanToFuture(EntitlementRequestAlreadyExists, cc = Some(cc)) { - EntitlementRequest.entitlementRequest.vend.getEntitlementRequest(body.bank_id, user.userId, body.role_name).isEmpty + EntitlementRequest.entitlementRequest.vend.getEntitlementRequest(body.bank_id, requesterUserId, body.role_name).isEmpty } - addedEntitlementRequest <- EntitlementRequest.entitlementRequest.vend.addEntitlementRequestFuture(body.bank_id, user.userId, body.role_name) map { + addedEntitlementRequest <- EntitlementRequest.entitlementRequest.vend.addEntitlementRequestFuture(body.bank_id, requesterUserId, body.role_name) map { x => unboxFullOrFail(x, Some(cc), EntitlementRequestCannotBeAdded) } } yield JSONFactory300.createEntitlementRequestJSON(addedEntitlementRequest) diff --git a/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala b/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala index 168094cbc3..c6c8b4e74d 100644 --- a/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala +++ b/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala @@ -4301,10 +4301,16 @@ object Http4s310 { (accountBox, _) <- Connector.connector.vend.checkBankAccountExists(bank.bankId, AccountId(accountIdStr), Some(cc)) _ <- code.util.Helper.booleanToFuture(AccountIdAlreadyExists, cc = Some(cc)) { accountBox.isEmpty } loggedInUserId = user.userId - userIdAccountOwner = if (body.user_id.nonEmpty) body.user_id else loggedInUserId + // Implicit owner resolves to the HUMAN: under a Consent the caller is the + // per-consent shadow, and an account held by it strands when the consent dies. + userIdAccountOwner = if (body.user_id.nonEmpty) body.user_id else cc.accountableUserId _ <- code.util.Helper.booleanToFuture(InvalidAccountIdFormat, cc = Some(cc)) { isValidID(accountIdStr) } _ <- code.util.Helper.booleanToFuture(InvalidBankIdFormat, cc = Some(cc)) { isValidID(bankIdStr) } (accountOwner, _) <- NewStyle.function.findByUserId(userIdAccountOwner, Some(cc)) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + _ <- code.util.Helper.booleanToFuture( + s"$InvalidUserId user_id names a consent user (an agent identity minted by a Consent). Accounts are held by humans - use the granting user's USER_ID.", + failCode = 400, cc = Some(cc))(!accountOwner.isConsentUser) _ <- if (userIdAccountOwner == loggedInUserId) Future.successful(Full(())) else code.util.Helper.booleanToFuture( s"$UserHasMissingRoles $canCreateAccount or create account for self", diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala index d821649e0d..7b61eba6a3 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala @@ -10137,10 +10137,16 @@ object Http4s400 { com.openbankproject.commons.util.JsonAliases.parse(rawBody).extract[code.api.v3_1_0.CreateAccountRequestJsonV310] } loggedInUserId = cc.userId + // Implicit owner resolves to the HUMAN: under a Consent the caller is the + // per-consent shadow, and an account held by it strands when the consent dies. userIdAccountOwner = if (createAccountJson.user_id.nonEmpty) createAccountJson.user_id - else loggedInUserId + else cc.accountableUserId (postedOrLoggedInUser, callContext) <- NewStyle.function.findByUserId(userIdAccountOwner, Some(cc)) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + _ <- code.util.Helper.booleanToFuture( + s"$InvalidUserId user_id names a consent user (an agent identity minted by a Consent). Accounts are held by humans - use the granting user's USER_ID.", + failCode = 400, cc = Some(cc))(!postedOrLoggedInUser.isConsentUser) _ <- if (userIdAccountOwner == loggedInUserId) Future.successful(Full(())) else NewStyle.function.hasEntitlement( bankId.value, loggedInUserId, canCreateAccount, callContext, @@ -10207,10 +10213,16 @@ object Http4s400 { com.openbankproject.commons.util.JsonAliases.parse(rawBody).extract[SettlementAccountRequestJson] } loggedInUserId = cc.userId + // Implicit owner resolves to the HUMAN: under a Consent the caller is the + // per-consent shadow, and an account held by it strands when the consent dies. userIdAccountOwner = if (createAccountJson.user_id.nonEmpty) createAccountJson.user_id - else loggedInUserId + else cc.accountableUserId (postedOrLoggedInUser, callContext) <- NewStyle.function.findByUserId(userIdAccountOwner, Some(cc)) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + _ <- code.util.Helper.booleanToFuture( + s"$InvalidUserId user_id names a consent user (an agent identity minted by a Consent). Accounts are held by humans - use the granting user's USER_ID.", + failCode = 400, cc = Some(cc))(!postedOrLoggedInUser.isConsentUser) _ <- if (userIdAccountOwner == loggedInUserId) Future.successful(Full(())) else NewStyle.function.hasEntitlement(bankId.value, loggedInUserId, canCreateSettlementAccountAtOneBank, callContext) initialBalanceAsString = createAccountJson.balance.amount diff --git a/obp-api/src/main/scala/code/api/v5_0_0/Http4s500.scala b/obp-api/src/main/scala/code/api/v5_0_0/Http4s500.scala index 1de94f93d0..97fe6af541 100644 --- a/obp-api/src/main/scala/code/api/v5_0_0/Http4s500.scala +++ b/obp-api/src/main/scala/code/api/v5_0_0/Http4s500.scala @@ -463,18 +463,21 @@ object Http4s500 { postJson.bank_routings.getOrElse(Nil).filterNot(_.scheme == "BIC").headOption.map(_.address).getOrElse(""), Some(cc) ) - entitlements <- NewStyle.function.getEntitlementsByUserId(cc.userId, Some(cc)) + // Creator grants target the HUMAN (see v6.0.0 createBank): under a Consent the + // authenticated user is a per-consent shadow, and roles granted to it are stranded. + humanUserId = cc.accountableUserId + entitlements <- NewStyle.function.getEntitlementsByUserId(humanUserId, Some(cc)) entitlementsByBank = entitlements.filter(_.bankId == postJson.id.getOrElse("")) _ <- entitlementsByBank.exists(_.roleName == CanCreateEntitlementAtOneBank.toString()) match { case true => Future.successful(()) case false => Future(Entitlement.entitlement.vend.addEntitlement( - postJson.id.getOrElse(""), cc.userId, CanCreateEntitlementAtOneBank.toString(), + postJson.id.getOrElse(""), humanUserId, CanCreateEntitlementAtOneBank.toString(), grantedByUserId = Some(cc.userId))) } _ <- entitlementsByBank.exists(_.roleName == CanReadDynamicResourceDocsAtOneBank.toString()) match { case true => Future.successful(()) case false => Future(Entitlement.entitlement.vend.addEntitlement( - postJson.id.getOrElse(""), cc.userId, CanReadDynamicResourceDocsAtOneBank.toString(), + postJson.id.getOrElse(""), humanUserId, CanReadDynamicResourceDocsAtOneBank.toString(), grantedByUserId = Some(cc.userId))) } } yield JSONFactory500.createBankJSON500(success) @@ -584,10 +587,16 @@ object Http4s500 { com.openbankproject.commons.util.JsonAliases.parse(cc.httpBody.getOrElse("")).extract[CreateAccountRequestJsonV500] } loggedInUserId = user.userId - userIdAccountOwner = createAccountJson.user_id.getOrElse(loggedInUserId) + // Implicit owner resolves to the HUMAN: under a Consent the caller is the + // per-consent shadow, and an account held by it strands when the consent dies. + userIdAccountOwner = createAccountJson.user_id.getOrElse(cc.accountableUserId) _ <- Helper.booleanToFuture(InvalidAccountIdFormat, cc = Some(cc)) { isValidID(accountId.value) } _ <- Helper.booleanToFuture(InvalidBankIdFormat, cc = Some(cc)) { isValidID(accountId.value) } (postedOrLoggedInUser, _) <- NewStyle.function.findByUserId(userIdAccountOwner, Some(cc)) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + _ <- Helper.booleanToFuture( + s"$InvalidUserId user_id names a consent user (an agent identity minted by a Consent). Accounts are held by humans - use the granting user's USER_ID.", + failCode = 400, cc = Some(cc))(!postedOrLoggedInUser.isConsentUser) _ <- if (userIdAccountOwner == loggedInUserId) Future.successful(Full(())) else Helper.booleanToFuture( s"${UserHasMissingRoles} $canCreateAccount", failCode = 403, cc = Some(cc)) { diff --git a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala index 96e1d695d0..009a7ae9ee 100644 --- a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala +++ b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala @@ -3195,6 +3195,12 @@ object Http4s510 { APIUtil.canGrantAccessToView(com.openbankproject.commons.model.BankIdAccountIdViewId(bankId, accountId, viewId), targetViewId, user, Some(cc)) } (targetUser, _) <- NewStyle.function.findByUserId(postJson.user_id, Some(cc)) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + // A consent user's account access comes ONLY from its Consent (materialised and + // revoked with it); access granted here would outlive nothing and confuse audits. + _ <- Helper.booleanToFuture( + s"$InvalidUserId user_id names a consent user (an agent identity minted by a Consent). Account access targets humans - a consent user's access comes only from its Consent.", + failCode = 400, cc = Some(cc))(!targetUser.isConsentUser) view <- if (isValidSystemViewId(targetViewId.value)) ViewNewStyle.systemView(targetViewId, Some(cc)) else ViewNewStyle.customView(targetViewId, BankIdAccountId(bankId, accountId), Some(cc)) addedView <- JSONFactory400.grantAccountAccessToUser(bankId, accountId, targetUser, view, Some(cc)) diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index ff0f80087e..f7c1507713 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -199,7 +199,6 @@ object Http4s600 { else "oidc_operator_user_ids" def entitlementRequestId: Option[String] = None def groupId: Option[String] = None - def process: Option[String] = None def grantedByUserId: Option[String] = None } } @@ -491,8 +490,10 @@ object Http4s600 { else Nil ) } yield { + // Creator grants target the HUMAN (see createBank): a per-consent shadow principal + // must not end up owning the entity's admin roles. crudRoles.foreach(role => - Entitlement.entitlement.vend.addEntitlement(dynamicEntity.bankId.getOrElse(""), cc.userId, role.toString(), + Entitlement.entitlement.vend.addEntitlement(dynamicEntity.bankId.getOrElse(""), cc.accountableUserId, role.toString(), grantedByUserId = Some(cc.userId))) JSONFactory600.createMyDynamicEntitiesJson(List(result: DynamicEntityCommons)).dynamic_entities.head } @@ -869,10 +870,15 @@ object Http4s600 { postJson.bank_routings.getOrElse(Nil).filterNot(_.scheme == "BIC").headOption.map(_.address).getOrElse(""), Some(cc) ) - entitlements <- NewStyle.function.getEntitlementsByUserId(cc.userId, Some(cc)) + // Creator grant goes to the HUMAN, not the authenticated principal: under a + // Consent the principal is a per-consent shadow user, and a role granted to it + // is stranded when the consent dies (and invisible to the human's next consent). + // grantedByUserId stays the principal — the audit trail records who acted. + humanUserId = cc.accountableUserId + entitlements <- NewStyle.function.getEntitlementsByUserId(humanUserId, Some(cc)) entitlementsByBank = entitlements.filter(_.bankId == postJson.bank_id) _ = if (!entitlementsByBank.exists(_.roleName == CanCreateEntitlementAtOneBank.toString)) - Entitlement.entitlement.vend.addEntitlement(postJson.bank_id, cc.userId, CanCreateEntitlementAtOneBank.toString, + Entitlement.entitlement.vend.addEntitlement(postJson.bank_id, humanUserId, CanCreateEntitlementAtOneBank.toString, grantedByUserId = Some(cc.userId)) } yield JSONFactory600.createBankJSON600(success) } @@ -1943,7 +1949,14 @@ object Http4s600 { postJson <- NewStyle.function.tryons(s"$InvalidJsonFormat The Json body should be the PostGroupMembershipJsonV600", 400, Some(cc)) { com.openbankproject.commons.util.JsonAliases.parse(rawBody).extract[JSONFactory600.PostGroupMembershipJsonV600] } - _ <- NewStyle.function.findByUserId(userIdStr, Some(cc)) + (targetUser, _) <- NewStyle.function.findByUserId(userIdStr, Some(cc)) + // Group membership is for humans. A consent user (an agent identity minted by a + // Consent) cannot hold durable roles — addEntitlement would redirect the grant to + // its granting human anyway, and removal via the consent user's id would then find + // nothing. Reject explicitly so the caller targets the human on purpose. + _ <- Helper.booleanToFuture( + s"$InvalidUserId USER_ID names a consent user (an agent identity minted by a Consent). Group membership targets humans - use the granting user's USER_ID.", + 400, Some(cc))(!targetUser.isConsentUser) group <- Future(code.group.GroupTrait.group.vend.getGroup(postJson.group_id)) .map(unboxFullOrFail(_, Some(cc), s"$UnknownError Group not found", 404)) _ <- groupRoleCheck(group.bankId, user.userId, canAddUserToGroupAtOneBank, canAddUserToGroupAtAllBanks, cc) @@ -1955,9 +1968,12 @@ object Http4s600 { ent.roleName == roleName && ent.bankId == group.bankId.getOrElse("") }) if (!alreadyHas) { + // createdByProcess carries the provenance (was left at "manual", making + // group-born rows read as hand-granted before the duplicate `process` + // column was retired). Entitlement.entitlement.vend.addEntitlement( - group.bankId.getOrElse(""), userIdStr, roleName, "manual", - Some(user.userId), Some(postJson.group_id), Some("GROUP_MEMBERSHIP")) + group.bankId.getOrElse(""), userIdStr, roleName, Constant.group_membership, + Some(user.userId), Some(postJson.group_id)) (roleName, true) } else (roleName, false) } @@ -1983,8 +1999,10 @@ object Http4s600 { .map(unboxFullOrFail(_, Some(cc), s"$UnknownError Group not found", 404)) _ <- groupRoleCheck(group.bankId, user.userId, canRemoveUserFromGroupAtOneBank, canRemoveUserFromGroupAtAllBanks, cc) entitlements <- Future(Entitlement.entitlement.vend.getEntitlementsByUserId(userIdStr)) + // group_id alone identifies group-born rows (only group grants set it) and holds + // for legacy rows too; the old `process == GROUP_MEMBERSHIP` conjunct was redundant. groupEntitlements = entitlements.toOption.getOrElse(List.empty).filter(e => - e.groupId == Some(groupId) && e.process == Some("GROUP_MEMBERSHIP")) + e.groupId == Some(groupId)) _ <- Future.sequence(groupEntitlements.map(e => Future(Entitlement.entitlement.vend.deleteEntitlement(Full(e))))) } yield "" @@ -2485,7 +2503,13 @@ object Http4s600 { _ <- Helper.booleanToFuture(BusinessJustificationRequired, cc = Some(cc)) { postJson.business_justification.trim.nonEmpty } - (_, _) <- NewStyle.function.findByUserId(postJson.target_user_id, Some(cc)) + (targetUser, _) <- NewStyle.function.findByUserId(postJson.target_user_id, Some(cc)) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + // Reject at request creation so no approver ever sees a request that the grant + // step would refuse anyway. + _ <- Helper.booleanToFuture( + s"$InvalidUserId target_user_id names a consent user (an agent identity minted by a Consent). Account access targets humans - a consent user's access comes only from its Consent.", + failCode = 400, cc = Some(cc))(!targetUser.isConsentUser) _ <- Helper.booleanToFuture(AccountAccessRequestAlreadyExists, 409, Some(cc)) { code.accountaccessrequest.AccountAccessRequestTrait.accountAccessRequest.vend .getByUserAccountView(postJson.target_user_id, bankIdStr, accountIdStr, postJson.view_id) @@ -2535,6 +2559,11 @@ object Http4s600 { u.userId != request.requestorUserId } (targetUser, _) <- NewStyle.function.findByUserId(request.targetUserId, Some(cc)) + // Belt and braces with the creation-side guard: a request stored before that guard + // existed (or written another way) must still not be granted to a consent user. + _ <- Helper.booleanToFuture( + s"$InvalidUserId The request's target user is a consent user (an agent identity minted by a Consent). Account access targets humans - a consent user's access comes only from its Consent.", + failCode = 400, cc = Some(cc))(!targetUser.isConsentUser) // Win the INITIATED -> APPROVED transition BEFORE granting view access. The provider's // conditional UPDATE makes this request the single actioner; the loser of a concurrent // approve/reject race gets a 400 here with NO side effect. Granting first would leave @@ -4453,7 +4482,8 @@ object Http4s600 { for { (_, _) <- NewStyle.function.findByUserId(userId, Some(cc)) entitlements <- Future(code.entitlement.Entitlement.entitlement.vend.getEntitlementsByUserId(userId)) - groupEntitlements = entitlements.toOption.getOrElse(List.empty).filter(_.process == Some("GROUP_MEMBERSHIP")) + // group_id alone identifies group-born rows (see removeUserFromGroup). + groupEntitlements = entitlements.toOption.getOrElse(List.empty).filter(_.groupId.isDefined) groupIds = groupEntitlements.flatMap(_.groupId).distinct _ <- Future.sequence { groupIds.flatMap { gid => @@ -5607,7 +5637,7 @@ object Http4s600 { entitlement_id = ent.entitlementId, role_name = ent.roleName, bank_id = ent.bankId, user_id = ent.userId, username = userBox.map(_.name).getOrElse(""), - group_id = ent.groupId, process = ent.process) + group_id = ent.groupId, created_by_process = ent.createdByProcess) } }) } yield GroupEntitlementsJsonV600(withUsernames) @@ -9358,7 +9388,6 @@ object Http4s600 { | |Only removes entitlements with: |- group_id matching GROUP_ID - |- process = "GROUP_MEMBERSHIP" | |Requires either: |- CanRemoveUserFromGroupAtAllBanks (for any group) @@ -12527,7 +12556,7 @@ object Http4s600 { "Get User's Group Memberships", s"""Get all groups a user is a member of. | - |Returns groups where the user has entitlements with process = "GROUP_MEMBERSHIP". + |Returns groups where the user has entitlements carrying a group_id. | |The response includes: |- list_of_entitlements: entitlements the user currently has from this group membership @@ -13838,7 +13867,7 @@ object Http4s600 { user_id = "user-id-123", username = "susan.uk.29@example.com", group_id = Some("group-id-123"), - process = Some("GROUP_MEMBERSHIP") + created_by_process = "GROUP_MEMBERSHIP" ) ) ), diff --git a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala index a4961c34c0..0d126d57bf 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala @@ -472,6 +472,10 @@ case class MetricJsonV600( operation_id: String, api_instance_id: String, consent_reference_id: Option[String], + // Authentication scheme of the call: "Consent", "OAuth2", "OAuth1", "DirectLogin", + // "GatewayLogin", "DAuth", "Anonymous", "Other". Absent on rows written before the + // auth_type column existed. + auth_type: Option[String], // How the caller's certificate was established: "direct", "forwarded" or "none"; // absent when the request carried no certificate material. See PeerTrust.Resolution. certificate_trust: Option[String], @@ -1745,6 +1749,7 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { operation_id = operationId, api_instance_id = metric.getApiInstanceId(), consent_reference_id = Option(metric.getConsentReferenceId()).filter(_.nonEmpty), + auth_type = Option(metric.getAuthType()).filter(_.nonEmpty), certificate_trust = Option(metric.getCertificateTrust()).filter(_.nonEmpty), certificate_trust_detail = Option(metric.getCertificateTrustDetail()).filter(_.nonEmpty) ) @@ -2061,7 +2066,9 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { user_id: String, username: String, group_id: Option[String], - process: Option[String] + // The row's stored provenance, verbatim: "GROUP_MEMBERSHIP" for rows granted since + // provenance moved to created_by_process; legacy group rows show "manual". + created_by_process: String ) case class GroupEntitlementsJsonV600( diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala index 55f4c40f0b..f0b393193b 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala @@ -287,12 +287,12 @@ object Http4s700 { // Response shapes reuse the v6 bank JSON (BankJson600 / BanksJsonV600). // ─── Delegation fan-down for /my/banks ─────────────────────────────────── - // Resolving UP (agent caller → the granting human) is cc.effectiveHumanUserId. + // Resolving UP (agent caller → the granting human) is cc.accountableUserId. // This is the fan DOWN: the human plus every agent user minted from any Consent the // human granted — i.e. all user ids whose creations belong to that human. Match the // result against CreatedByUserId. Reads only server-written columns // (MappedConsent.mUserId, ResourceUser.CreatedByConsentId); the input must be an - // already-resolved human id (cc.effectiveHumanUserId), never a raw caller value. + // already-resolved human id (cc.accountableUserId), never a raw caller value. private def humanAndAgentUserIds(humanUserId: String): List[String] = { val consentIds = Consents.consentProvider.vend.getConsentsByUser(humanUserId) @@ -336,7 +336,7 @@ object Http4s700 { // Quota binds to the human: banks created by the human directly or by any // of their consent-agents count toward the same limit — otherwise every // new consent would arrive with a fresh quota. - val creatorUserIds = humanAndAgentUserIds(cc.effectiveHumanUserId) + val creatorUserIds = humanAndAgentUserIds(cc.accountableUserId) MappedBank.count(ByList(MappedBank.CreatedByUserId, creatorUserIds)) } _ <- Helper.booleanToFuture(SelfServiceBankLimitReached, failCode = 403, cc = Some(cc)) { @@ -356,8 +356,11 @@ object Http4s700 { "", "", "", "", "", "", Some(cc) ) + // Creator grant targets the HUMAN (see v6.0.0 createBank): under a Consent the + // authenticated user is a per-consent shadow, and roles granted to it are stranded. _ <- Future(Entitlement.entitlement.vend.addEntitlement( - generatedName.bankId, cc.userId, canCreateEntitlementAtOneBank.toString())) + generatedName.bankId, cc.accountableUserId, canCreateEntitlementAtOneBank.toString(), + grantedByUserId = Some(cc.userId))) } yield JSONFactory600.createBankJSON600(bank) } } @@ -414,7 +417,7 @@ object Http4s700 { EndpointHelpers.withUser(req) { (user, cc) => for { banksCreatedByUser <- Future { - val creatorUserIds = humanAndAgentUserIds(cc.effectiveHumanUserId) + val creatorUserIds = humanAndAgentUserIds(cc.accountableUserId) MappedBank.findAll(ByList(MappedBank.CreatedByUserId, creatorUserIds)) } } yield JSONFactory600.createBanksJsonV600(banksCreatedByUser) @@ -484,7 +487,13 @@ object Http4s700 { case req @ POST -> `prefixPath` / "users" / userId / "entitlements" => EndpointHelpers.withUserAndBodyCreated[CreateEntitlementJSON, AnyRef](req) { (user, body, cc) => for { - (_, _) <- NewStyle.function.findByUserId(userId, Some(cc)) + (targetUser, _) <- NewStyle.function.findByUserId(userId, Some(cc)) + // Explicit target: fail loud rather than redirect. A consent user (an agent + // identity minted by a Consent) cannot hold durable roles — grant to the + // granting human instead. + _ <- Helper.booleanToFuture( + s"$InvalidUserId USER_ID names a consent user (an agent identity minted by a Consent). Entitlements target humans - use the granting user's USER_ID.", + failCode = 400, cc = Some(cc))(!targetUser.isConsentUser) role <- NewStyle.function.tryons( s"$InvalidJsonFormat Unknown role: ${body.role_name}. Possible roles: ${ApiRole.availableRoles.sorted.mkString(", ")}", 400, Some(cc)) { ApiRole.valueOf(body.role_name) } @@ -933,7 +942,6 @@ object Http4s700 { else "oidc_operator_user_ids" def entitlementRequestId: Option[String] = None def groupId: Option[String] = None - def process: Option[String] = None def grantedByUserId: Option[String] = None } } @@ -1140,7 +1148,7 @@ object Http4s700 { // human, then fan down — both via server-written columns only. (metrics, _) <- APIMetrics.getMetricsFromHttpParams( httpParams, cc.callContext, - lockedUserIds = Some(humanAndAgentUserIds(cc.effectiveHumanUserId))) + lockedUserIds = Some(humanAndAgentUserIds(cc.accountableUserId))) } yield JSONFactory600.createMetricsJsonV600(metrics) } } @@ -4197,8 +4205,14 @@ object Http4s700 { case None => Future.successful(AccountId(APIUtil.generateUUID())) } // CanCreateAccount is enforced by ResourceDocMiddleware from the doc. - ownerId = body.user_id.filter(_.trim.nonEmpty).getOrElse(user.userId) + // The implicit owner is the HUMAN: under a Consent the caller (user.userId) is the + // per-consent shadow, and an account held by it strands when the consent dies. + ownerId = body.user_id.filter(_.trim.nonEmpty).getOrElse(cc.accountableUserId) (owner, _) <- NewStyle.function.findByUserId(ownerId, Some(cc)) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + _ <- Helper.booleanToFuture( + s"$InvalidUserId user_id names a consent user (an agent identity minted by a Consent). Accounts are held by humans - use the granting user's USER_ID.", + failCode = 400, cc = Some(cc))(!owner.isConsentUser) initialBalance <- NewStyle.function.tryons(InvalidAccountInitialBalance, 400, Some(cc)) { BigDecimal(body.balance.amount) } diff --git a/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnectorInternal.scala b/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnectorInternal.scala index 9d05ca346c..fae816abb6 100644 --- a/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnectorInternal.scala +++ b/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnectorInternal.scala @@ -1526,7 +1526,13 @@ object LocalMappedConnectorInternal extends MdcLoggable { accountRoutings = Nil, callContext = callContext ) - _ <- code.model.dataAccess.BankAccountCreation.setAccountHolderAndRefreshUserAccountAccess(bankId, newAccountId, cc.get.user.head, callContext) + // Holder is the HUMAN: under a Consent cc.user is the per-consent shadow, and a + // holding account held by it would strand when the consent dies. For non-consent + // callers accountableUserId is the caller, so this is a no-op for them. + holdingAccountHolder = cc.flatMap(c => + code.users.Users.users.vend.getUserByUserId(c.accountableUserId).toOption + ).getOrElse(cc.get.user.head) + _ <- code.model.dataAccess.BankAccountCreation.setAccountHolderAndRefreshUserAccountAccess(bankId, newAccountId, holdingAccountHolder, callContext) // create attribute on holding account to link to releaser account _ <- NewStyle.function.createOrUpdateAccountAttribute( bankId = bankId, diff --git a/obp-api/src/main/scala/code/entitlement/Entilement.scala b/obp-api/src/main/scala/code/entitlement/Entilement.scala index 211d227f0a..9fbe78ae4d 100644 --- a/obp-api/src/main/scala/code/entitlement/Entilement.scala +++ b/obp-api/src/main/scala/code/entitlement/Entilement.scala @@ -44,8 +44,7 @@ trait EntitlementProvider { // createdByProcess carries the provenance. Authorization is the // calling endpoint's responsibility, not this method's. grantedByUserId: Option[String] = None, - groupId: Option[String] = None, - process: Option[String] = None + groupId: Option[String] = None ): Box[Entitlement] def deleteDynamicEntityEntitlement( entityName: String, @@ -62,7 +61,6 @@ trait Entitlement { def createdByProcess: String def entitlementRequestId: Option[String] def groupId: Option[String] - def process: Option[String] /** user_id of the granter, when the grant was made by a person (directly * or as a self-grant). None for system-process grants and virtual diff --git a/obp-api/src/main/scala/code/entitlement/MappedEntitlements.scala b/obp-api/src/main/scala/code/entitlement/MappedEntitlements.scala index da3c09b385..7d032cb521 100644 --- a/obp-api/src/main/scala/code/entitlement/MappedEntitlements.scala +++ b/obp-api/src/main/scala/code/entitlement/MappedEntitlements.scala @@ -6,6 +6,7 @@ import code.api.util.ApiRole.{ CanCreateEntitlementAtOneBank } import code.api.util.{ErrorMessages, NotificationUtil} +import code.util.Helper.MdcLoggable import code.util.{MappedUUID, UUIDString} import net.liftweb.common.{Box, Failure, Full} import net.liftweb.mapper._ @@ -15,7 +16,7 @@ import scala.concurrent.Future import com.openbankproject.commons.ExecutionContext.Implicits.global import net.liftweb.common -object MappedEntitlementsProvider extends EntitlementProvider { +object MappedEntitlementsProvider extends EntitlementProvider with MdcLoggable { override def getEntitlement( bankId: String, userId: String, @@ -163,32 +164,57 @@ object MappedEntitlementsProvider extends EntitlementProvider { roleName: String, createdByProcess: String = "manual", grantedByUserId: Option[String] = None, - groupId: Option[String] = None, - process: Option[String] = None + groupId: Option[String] = None ): Box[Entitlement] = { // grantedByUserId is audit metadata, stored as-is: authorization is the // calling endpoint's responsibility. (Until 2026-08-09 an unused // grantorUserId parameter gated on the grantor's granting roles here — // no caller ever passed it, and the check ignored super admins, whose // granting rights are virtual and have no rows to find.) + + // On-behalf-of guard: a consent user (the per-consent principal a Consent-JWT + // authenticates as; its ResourceUser row carries CreatedByConsentId) must not + // accumulate durable roles — they strand when the consent dies, invisible to the + // human's next consent (see the simon.bank creator-grant incident, 2026-08-31). + // Any grant targeting one is redirected to the consent's granting human. The one + // legitimate writer of consent-user rows is the consent engine copying the + // consent's own scope, which tags itself Constant.consent_user and is exempt. + val targetUserId = + if (createdByProcess == code.api.Constant.consent_user) userId + else { + val grantingHumanUserId = for { + resourceUser <- code.model.dataAccess.ResourceUser.find( + By(code.model.dataAccess.ResourceUser.userId_, userId)) + consentId <- Full(resourceUser.CreatedByConsentId.get) + .filter(id => id != null && id.nonEmpty) + consent <- code.consent.Consents.consentProvider.vend.getConsentByConsentId(consentId) + humanUserId <- Full(consent.userId).filter(id => id != null && id.nonEmpty) + } yield humanUserId + grantingHumanUserId match { + case Full(humanUserId) => + logger.warn(s"addEntitlement: target user $userId is a consent user; granting role '$roleName' (bankId '$bankId', createdByProcess '$createdByProcess') to its granting human $humanUserId instead") + humanUserId + case _ => userId + } + } + def addEntitlementToUser(): Box[MappedEntitlement] = { val entitlement = MappedEntitlement.create .mBankId(bankId) - .mUserId(userId) + .mUserId(targetUserId) .mRoleName(roleName) .mCreatedByProcess(createdByProcess) grantedByUserId.foreach(g => entitlement.mGrantedByUserId(g)) groupId.foreach(gid => entitlement.mGroupId(gid)) - process.foreach(p => entitlement.mProcess(p)) tryo(entitlement.saveMe()) match { case Full(saved) => - NotificationUtil.sendEmailRegardingAssignedRole(userId, saved) + NotificationUtil.sendEmailRegardingAssignedRole(targetUserId, saved) Full(saved) case Failure(_, _, _) => // UniqueIndex(mBankId, mUserId, mRoleName) violated by concurrent grant — return the committed row MappedEntitlement.find( By(MappedEntitlement.mBankId, bankId), - By(MappedEntitlement.mUserId, userId), + By(MappedEntitlement.mUserId, targetUserId), By(MappedEntitlement.mRoleName, roleName) ) case other => other @@ -217,10 +243,10 @@ class MappedEntitlement override def defaultValue = "" } - object mProcess extends MappedString(this, 255) { - override def dbColumnName = "process" - override def defaultValue = "" - } + // The legacy "process" DB column (a duplicate of createdByProcess written only by the + // Groups feature) is no longer mapped: group rows are identified by group_id, and + // provenance lives in createdByProcess. The column itself can be dropped from the DB + // whenever convenient. object entitlement_request_id extends MappedUUID(this) { override def dbColumnName = "entitlement_request_id" @@ -243,10 +269,6 @@ class MappedEntitlement val gid = mGroupId.get if (gid == null || gid.isEmpty) None else Some(gid) } - override def process: Option[String] = { - val p = mProcess.get - if (p == null || p.isEmpty) None else Some(p) - } override def grantedByUserId: Option[String] = { val g = mGrantedByUserId.get if (g == null || g.isEmpty) None else Some(g) diff --git a/obp-api/src/main/scala/code/metrics/APIMetrics.scala b/obp-api/src/main/scala/code/metrics/APIMetrics.scala index c8b26ae5e8..c3dfa8e784 100644 --- a/obp-api/src/main/scala/code/metrics/APIMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/APIMetrics.scala @@ -101,7 +101,8 @@ trait APIMetrics { apiInstanceId: String, consentReferenceId: String, certificateTrust: String, - certificateTrustDetail: String): Unit + certificateTrustDetail: String, + authType: String): Unit def saveMetricsArchive(primaryKey: Long, userId: String, @@ -123,7 +124,8 @@ trait APIMetrics { apiInstanceId: String, consentReferenceId: String, certificateTrust: String, - certificateTrustDetail: String + certificateTrustDetail: String, + authType: String ): Boolean // //TODO: ordering of list? should this be by date? currently not enforced @@ -183,6 +185,9 @@ trait APIMetric { def getConsentReferenceId(): String def getCertificateTrust(): String def getCertificateTrustDetail(): String + // Authentication scheme of the call — "Consent", "OAuth2", "OAuth1", "DirectLogin", + // "GatewayLogin", "DAuth", "Anonymous" or "Other". Scheme only, never credentials. + def getAuthType(): String } @@ -210,7 +215,7 @@ case class AggregateMetrics( minResponseTime: Double, maxResponseTime: Double, // Distinct humans behind the calls: consent-borne rows are attributed to the granting - // (on-behalf-of) user via the consent table, mirroring CallContext.effectiveHumanUserId. + // (on-behalf-of) user via the consent table, mirroring CallContext.accountableUserId. distinctUserCount: Int, distinctConsumerCount: Int, // Calls that arrived under a consent (metric.consent_reference_id not null), and how many diff --git a/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala b/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala index fb80d918d3..925cb6abed 100644 --- a/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala @@ -15,7 +15,8 @@ object ElasticsearchMetrics extends APIMetrics { override def saveMetric(userId: String, url: String, date: Date, duration: Long, userName: String, appName: String, developerEmail: String, consumerId: String, implementedByPartialFunction: String, implementedInVersion: String, verb: String, httpCode: Option[Int], correlationId: String, responseBody: String, sourceIp: String, targetIp: String, apiInstanceId: String, consentReferenceId: String, - certificateTrust: String, certificateTrustDetail: String): Unit = { + certificateTrust: String, certificateTrustDetail: String, + authType: String): Unit = { if (APIUtil.getPropsAsBoolValue("allow_elasticsearch", false) && APIUtil.getPropsAsBoolValue("allow_elasticsearch_metrics", false) ) { //TODO ,need to be fixed now add more parameters es.indexMetric(userId, url, date, duration, userName, appName, developerEmail, correlationId, apiInstanceId) @@ -28,7 +29,8 @@ object ElasticsearchMetrics extends APIMetrics { apiInstanceId: String, consentReferenceId: String, certificateTrust: String, - certificateTrustDetail: String): Boolean = ??? + certificateTrustDetail: String, + authType: String): Boolean = ??? // override def getAllGroupedByUserId(): Map[String, List[APIMetric]] = { // //TODO: replace the following with valid ES query diff --git a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala index e890e5e2e6..e6ad043619 100644 --- a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala @@ -114,7 +114,8 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ override def saveMetric(userId: String, url: String, date: Date, duration: Long, userName: String, appName: String, developerEmail: String, consumerId: String, implementedByPartialFunction: String, implementedInVersion: String, verb: String, httpCode: Option[Int], correlationId: String, responseBody: String, sourceIp: String, targetIp: String, apiInstanceId: String, consentReferenceId: String, - certificateTrust: String, certificateTrustDetail: String): Unit = { + certificateTrust: String, certificateTrustDetail: String, + authType: String): Unit = { // A correlation id is expected on every metric. Rows without one cannot be moved // to the archive later (its correlationId column requires a UUID), so flag it at // write time where the source of the missing id can actually be traced. @@ -142,7 +143,8 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ apiInstanceId = apiInstanceId, consentReferenceId = consentReferenceId, certificateTrust = certificateTrust, - certificateTrustDetail = certificateTrustDetail + certificateTrustDetail = certificateTrustDetail, + authType = authType ) ) } @@ -153,7 +155,8 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ verb: String, httpCode: Option[Int], correlationId: String, responseBody: String, sourceIp: String, targetIp: String, apiInstanceId: String, consentReferenceId: String, - certificateTrust: String, certificateTrustDetail: String): Boolean = { + certificateTrust: String, certificateTrustDetail: String, + authType: String): Boolean = { // Fix: dedup by the source metric's primary key stored in `metricId`, NOT by the // archive's own auto-increment `id`. The two are unrelated id-spaces; matching on // `id` overwrites an unrelated archived row once the archive's id sequence grows @@ -181,6 +184,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ .consentReferenceId(consentReferenceId) .certificateTrust(certificateTrust) .certificateTrustDetail(certificateTrustDetail) + .authType(authType) httpCode match { case Some(code) => metric.httpCode(code) @@ -431,7 +435,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ // The LEFT JOIN attributes consent-borne calls to the granting (on-behalf-of) human: // metric.userid records the AUTHENTICATED principal, which under a consent is the // consent's own shadow user. COALESCE(consent.muserid, metric.userid) resolves such rows - // to the granting human at read time, mirroring CallContext.effectiveHumanUserId. (Rows + // to the granting human at read time, mirroring CallContext.accountableUserId. (Rows // written 2026-08 only, while toLight briefly recorded the human, resolve identically.) // The consent side of the join is unique-indexed on consent_reference_id, so the join // cannot fan out rows. @@ -827,6 +831,13 @@ class MappedMetric extends APIMetric with LongKeyedMapper[MappedMetric] with IdP } // The specifics behind certificateTrust (PeerTrust.Resolution.detail): the forwarding proxy's // canonical subject DN for "forwarded", the rejection reason for "none". Null for "direct". + // Authentication scheme of the call (scheme only, never credentials): "Consent", + // "OAuth2", "OAuth1", "DirectLogin", "GatewayLogin", "DAuth", "Anonymous", "Other". + // Null on rows written before the column existed. + object authType extends MappedString(this, 32) { + override def dbColumnName = "auth_type" + override def defaultValue = null + } object certificateTrustDetail extends MappedString(this, 255) { override def dbColumnName = "certificate_trust_detail" override def defaultValue = null @@ -853,6 +864,7 @@ class MappedMetric extends APIMetric with LongKeyedMapper[MappedMetric] with IdP override def getConsentReferenceId(): String = consentReferenceId.get override def getCertificateTrust(): String = certificateTrust.get override def getCertificateTrustDetail(): String = certificateTrustDetail.get + override def getAuthType(): String = authType.get } object MappedMetric extends MappedMetric with LongKeyedMetaMapper[MappedMetric] { @@ -921,6 +933,13 @@ class MetricArchive extends APIMetric with LongKeyedMapper[MetricArchive] with I override def dbColumnName = "certificate_trust" override def defaultValue = null } + // Authentication scheme of the call (scheme only, never credentials): "Consent", + // "OAuth2", "OAuth1", "DirectLogin", "GatewayLogin", "DAuth", "Anonymous", "Other". + // Null on rows written before the column existed. + object authType extends MappedString(this, 32) { + override def dbColumnName = "auth_type" + override def defaultValue = null + } object certificateTrustDetail extends MappedString(this, 255) { override def dbColumnName = "certificate_trust_detail" override def defaultValue = null @@ -948,6 +967,7 @@ class MetricArchive extends APIMetric with LongKeyedMapper[MetricArchive] with I override def getConsentReferenceId(): String = consentReferenceId.get override def getCertificateTrust(): String = certificateTrust.get override def getCertificateTrustDetail(): String = certificateTrustDetail.get + override def getAuthType(): String = authType.get } object MetricArchive extends MetricArchive with LongKeyedMetaMapper[MetricArchive] { override def dbIndexes = diff --git a/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala b/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala index d1a05d5ba2..36107eafe1 100644 --- a/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala +++ b/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala @@ -43,7 +43,8 @@ object MetricBatchWriter extends MdcLoggable { apiInstanceId: String, consentReferenceId: String, certificateTrust: String, - certificateTrustDetail: String + certificateTrustDetail: String, + authType: String ) private val queue = new ConcurrentLinkedQueue[MetricRow]() @@ -106,8 +107,8 @@ object MetricBatchWriter extends MdcLoggable { developeremail, consumerid, implementedbypartialfunction, implementedinversion, verb, httpcode, correlationid, responsebody, sourceip, targetip, apiinstanceid, consent_reference_id, - certificate_trust, certificate_trust_detail - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + certificate_trust, certificate_trust_detail, auth_type + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """ // Use Option[String] so Doobie handles nullable fields via Put[Option[String]] @@ -117,7 +118,7 @@ object MetricBatchWriter extends MdcLoggable { Option[String], Option[String], Option[String], Option[String], Option[String], Int, Option[String], Option[String], Option[String], Option[String], Option[String], Option[String], - Option[String], Option[String]) + Option[String], Option[String], Option[String]) ](insertSql) val values = rows.map { r => @@ -128,7 +129,8 @@ object MetricBatchWriter extends MdcLoggable { Option(r.implementedInVersion), Option(r.verb), r.httpCode, Option(r.correlationId), Option(r.responseBody), Option(r.sourceIp), Option(r.targetIp), Option(r.apiInstanceId), Option(r.consentReferenceId), - Option(r.certificateTrust), Option(r.certificateTrustDetail) + Option(r.certificateTrust), Option(r.certificateTrustDetail), + Option(r.authType) ) } diff --git a/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala b/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala index 6697c04976..bb305ec7d2 100644 --- a/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala +++ b/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala @@ -149,7 +149,7 @@ class ResourceUser extends LongKeyedMapper[ResourceUser] with User with ManyToMa object ResourceUser extends ResourceUser with LongKeyedMetaMapper[ResourceUser]{ // userId_ is deliberately NOT declared here: MigrationOfUserIdIndexes creates a stronger // UNIQUE index on it (resourceuser_userid_unique). CreatedByConsentId is the delegation - // registry — consent-agent fan-down (/my/metrics, /my/banks) and effectiveHumanUserId join + // registry — consent-agent fan-down (/my/metrics, /my/banks) and accountableUserId join // through it; nothing else indexes it, which matters on consent-heavy instances where every // consent mints a user row. override def dbIndexes = UniqueIndex(provider_, providerId) :: Index(CreatedByConsentId) :: super.dbIndexes diff --git a/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala b/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala index 4633114f1b..e0faac79aa 100644 --- a/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala +++ b/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala @@ -227,7 +227,8 @@ object MetricsArchiveScheduler extends MdcLoggable { i.getApiInstanceId(), i.getConsentReferenceId(), i.getCertificateTrust(), - i.getCertificateTrustDetail() + i.getCertificateTrustDetail(), + i.getAuthType() ) } diff --git a/obp-api/src/test/scala/code/api/util/AgentDelegationTest.scala b/obp-api/src/test/scala/code/api/util/AgentDelegationTest.scala index 3171733e8b..571480a9a7 100644 --- a/obp-api/src/test/scala/code/api/util/AgentDelegationTest.scala +++ b/obp-api/src/test/scala/code/api/util/AgentDelegationTest.scala @@ -14,7 +14,7 @@ import org.scalatest.Tag * - LiftUsers.createResourceUser field assignments — pins the 2021 copy-paste bug where * the createdByUserInvitationId None branch wiped CreatedByConsentId (the consent → * agent linkage every delegation query joins through). - * - CallContext.effectiveHumanUserId — resolve-up from the authenticated caller (human + * - CallContext.accountableUserId — resolve-up from the authenticated caller (human * or consent-minted agent) to the human the request is really about, including the * branch an HTTP test cannot reach (the agent as the caller). */ @@ -71,23 +71,23 @@ class AgentDelegationTest extends ServerSetup { } } - feature("CallContext.effectiveHumanUserId resolves the caller to the human the request is about") { + feature("CallContext.accountableUserId resolves the caller to the human the request is about") { scenario("a plain human resolves to themselves", AgentDelegationTag) { val human = createUser() - CallContext(user = Full(human)).effectiveHumanUserId shouldBe human.userId + CallContext(user = Full(human)).accountableUserId shouldBe human.userId } scenario("a consent-minted agent resolves to the granting human", AgentDelegationTag) { val human = createUser() val consent = MappedConsent.create.mUserId(human.userId).saveMe() val agent = createUser(createdByConsentId = Some(consent.consentId)) - CallContext(user = Full(agent)).effectiveHumanUserId shouldBe human.userId + CallContext(user = Full(agent)).accountableUserId shouldBe human.userId } scenario("an agent with a dangling consent id falls back to itself (fails closed)", AgentDelegationTag) { val agent = createUser(createdByConsentId = Some(generateUUID())) - CallContext(user = Full(agent)).effectiveHumanUserId shouldBe agent.userId + CallContext(user = Full(agent)).accountableUserId shouldBe agent.userId } scenario("a populated consenter box wins over the DB chain", AgentDelegationTag) { @@ -96,7 +96,7 @@ class AgentDelegationTest extends ServerSetup { val agent = createUser(createdByConsentId = Some(consent.consentId)) val consenterHuman = createUser() CallContext(user = Full(agent), consenter = Full(consenterHuman)) - .effectiveHumanUserId shouldBe consenterHuman.userId + .accountableUserId shouldBe consenterHuman.userId } scenario("onBehalfOfUser wins over consenter", AgentDelegationTag) { @@ -107,7 +107,7 @@ class AgentDelegationTest extends ServerSetup { user = Full(agent), consenter = Full(consenterHuman), onBehalfOfUser = Full(explicitHuman) - ).effectiveHumanUserId shouldBe explicitHuman.userId + ).accountableUserId shouldBe explicitHuman.userId } } } diff --git a/obp-api/src/test/scala/code/api/v6_0_0/MetricAuthTypeTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/MetricAuthTypeTest.scala new file mode 100644 index 0000000000..1a79eb22f6 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v6_0_0/MetricAuthTypeTest.scala @@ -0,0 +1,60 @@ +package code.api.v6_0_0 + +import code.api.util.APIUtil.OAuth._ +import code.api.util.ApiRole.CanReadMetrics +import code.entitlement.Entitlement +import code.metrics.MetricBatchWriter +import com.openbankproject.commons.util.ApiVersion +import org.scalatest.Tag + +/** + * Tests the auth_type field on metric rows (v6.0.0): the authentication SCHEME of each + * call — never the credential. Uses the url-filter isolation pattern (write_metrics is + * JVM-wide; other suites also land rows). + * + * The harness's <@ (user1) authenticates via DirectLogin, so this suite's rows must + * carry auth_type "DirectLogin"; the invariant "Consent only with a consent reference" + * is asserted over whatever rows the filter returns. + */ +class MetricAuthTypeTest extends V600ServerSetup { + + object VersionOfApi extends Tag(ApiVersion.v6_0_0.toString) + object ApiEndpoint1 extends Tag("getMetrics") + + private val KnownAuthTypes = + Set("Consent", "OAuth2", "OAuth1", "DirectLogin", "GatewayLogin", "DAuth", "Anonymous", "Other") + + feature(s"test auth_type on metric rows version $VersionOfApi") { + scenario("Rows carry the scheme that authenticated them", ApiEndpoint1, VersionOfApi) { + setPropsValues("write_metrics" -> "true") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanReadMetrics.toString) + + val trafficUrl = "/obp/v5.1.0/banks" + val requestBanks = (v5_1_0_Request / "banks").GET <@ (user1) + (1 to 3).foreach(_ => makeGetRequest(requestBanks)) + + MetricBatchWriter.flush() + + When("We fetch the metric rows for the traffic url") + val request = (v6_0_0_Request / "management" / "metrics").GET <@ (user1) < + val authType = (m \ "auth_type").extractOpt[String] + withClue(s"row: $m ") { + authType.isDefined shouldBe true + KnownAuthTypes should contain(authType.get) + // Scheme/reference consistency: "Consent" implies a consent reference and vice versa. + val consentRef = (m \ "consent_reference_id").extractOpt[String] + (authType.get == "Consent") shouldBe consentRef.isDefined + } + } + metrics.map(m => (m \ "auth_type").extractOpt[String]).flatten.toSet shouldBe Set("DirectLogin") + } + } +} diff --git a/obp-api/src/test/scala/code/metrics/MetricsTest.scala b/obp-api/src/test/scala/code/metrics/MetricsTest.scala index 729abd8cdd..28f468ed59 100644 --- a/obp-api/src/test/scala/code/metrics/MetricsTest.scala +++ b/obp-api/src/test/scala/code/metrics/MetricsTest.scala @@ -65,7 +65,7 @@ class MetricsTest extends ServerSetup with WipeMetrics { scenario("We save a new API metric") { metrics.saveMetric(testUserId,testUrl1, day1, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) MetricBatchWriter.flush() val byUrl = metrics.getAllMetrics(List(OBPLimit(limit))).groupBy(_.getUrl()) @@ -83,16 +83,16 @@ class MetricsTest extends ServerSetup with WipeMetrics { scenario("Group all metrics by url") { metrics.saveMetric(testUserId, testUrl1, day1, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) metrics.saveMetric(testUserId, testUrl1, day1, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) metrics.saveMetric(testUserId, testUrl1, day2, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) metrics.saveMetric(testUserId, testUrl2, day2, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) MetricBatchWriter.flush() val byUrl = metrics.getAllMetrics(List(OBPLimit(limit1))).groupBy(_.getUrl()) @@ -113,16 +113,16 @@ class MetricsTest extends ServerSetup with WipeMetrics { scenario("Group all metrics by day") { metrics.saveMetric(testUserId, testUrl1, day1, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) metrics.saveMetric(testUserId, testUrl1, day1, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) metrics.saveMetric(testUserId, testUrl1, day2, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) metrics.saveMetric(testUserId, testUrl2, day2, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) MetricBatchWriter.flush() val byDay = metrics.getAllMetrics(List(OBPLimit(limit2))).groupBy(APIMetrics.getMetricDay) From 9b458b6d85cebe60537f8ebb3bf1f38e23abbc07 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 08:38:18 +0200 Subject: [PATCH 46/59] fix: don't let a cached compile result skip fresh dependency validation createJavaHttp4sEndpoint memoized the whole Box[Http4sEndpointIO] -- including whatever Validation.validateDependency decided -- keyed only by the exact method_body string. Once dynamic_code_compile_validate_enable and the whitelist became live-reloadable (previous commit), this became a bypass: compile a Java source while validation is off, then turn validation on and tighten the whitelist, then resubmit the identical source in a new create/update call -- the cached Full(...) from the first, unvalidated compile is returned directly, and validateDependency never runs a second time. Split the memoization so only the actual javax.tools.JavaCompiler invocation is cached (deterministic given the source, and the one genuinely expensive step); dependency validation now runs on every call, unmemoized, exactly like the Scala path's CompiledObjects.validateDependency() already does. Also corrected an overclaiming comment: the val-to-def change makes Validation.allowedRuntimePermissions itself always current, but Sandbox.sandbox(bankId) separately caches the whole Sandbox it builds per bankId, so a sandbox built before a permissions-prop change keeps the old snapshot. Left that cache alone -- SecurityManager enforcement is already a no-op on this JDK (JEP 486), so neither the stale nor the fresh permission list is actually enforced either way. DynamicResourceDocJavaSecurityValidationTest gains a regression scenario reproducing the exact bypass sequence: compile the malicious source once with validation off, enable strict validation, resubmit the identical source under a new doc, and assert it is still rejected. --- .../scala/code/api/util/DynamicUtil.scala | 70 +++++++++++++------ ...esourceDocJavaSecurityValidationTest.scala | 61 +++++++++++++--- 2 files changed, 100 insertions(+), 31 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 42a1537263..6c37de021b 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -44,7 +44,9 @@ object DynamicUtil extends MdcLoggable{ val toolBox: ToolBox[universe.type] = runtimeMirror(getClass.getClassLoader).mkToolBox() private val memoClassPool = new Memo[ClassLoader, ClassPool] - private val memoJavaHttp4sEndpoint = new Memo[String, Box[code.api.util.APIUtil.Http4sEndpointIO]] + // Caches only the compiled artifact (deterministic given the source string), never the + // validation outcome built on top of it -- see createJavaHttp4sEndpoint's doc comment. + private val memoJavaCompiledScript = new Memo[String, Box[ch.obermuhlner.scriptengine.java.JavaCompiledScript]] private def getClassPool(classLoader: ClassLoader) = memoClassPool.memoize(classLoader){ val cp = ClassPool.getDefault @@ -368,6 +370,14 @@ object DynamicUtil extends MdcLoggable{ // step, DynamicUtil.compileScalaCodeUnchecked, is already memoized by the exact source // string, so re-evaluating these on every call is a cache hit unless the underlying props // value actually changed. + // + // This makes allowedRuntimePermissions itself always current, but NOT everything downstream + // of it: Sandbox.sandbox(bankId) below separately caches the whole Sandbox it builds, keyed + // only by bankId -- so a bankId whose sandbox was already built keeps that snapshot of + // allowedRuntimePermissions until the process restarts, same staleness this def change fixed + // for validateDependency. Left as-is here because it's moot in practice: SecurityManager + // enforcement is already a no-op on this JVM (JEP 486, JDK 24+; see Sandbox's own comment), + // so neither the stale nor the fresh permission list is actually enforced. def dynamicCodeSandboxPermissions = APIUtil.getPropsValue("dynamic_code_sandbox_permissions", "[]").trim def scalaCodePermissioins = "List[java.security.Permission]"+dynamicCodeSandboxPermissions.replaceFirst("\\[","(").dropRight(1)+")" def permissions:Box[List[java.security.Permission]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodePermissioins) @@ -620,7 +630,7 @@ object DynamicUtil extends MdcLoggable{ */ def createJavaHttp4sEndpoint(methodBody: String): Box[code.api.util.APIUtil.Http4sEndpointIO] = if (!dynamicCodeExecutionEnabled) Failure(ErrorMessages.DynamicCodeExecutionDisabled) - else memoJavaHttp4sEndpoint.memoize("java-http4s-endpoint:" + methodBody) { + else { import cats.effect.IO import code.api.util.APIUtil.Http4sEndpointIO import com.openbankproject.commons.ExecutionContext.Implicits.global @@ -631,25 +641,38 @@ object DynamicUtil extends MdcLoggable{ import scala.jdk.CollectionConverters._ - // Real compile happens here (javax.tools.JavaCompiler via the JSR-223 "java" engine) — any - // Java syntax/type error surfaces as an exception, caught by this `Box tryo` and turned into - // a Failure. Deliberately narrow: only the compile step is wrapped. Validation runs below, - // outside any tryo — see the comment there for why that separation matters. - val compiledScriptBox: Box[ch.obermuhlner.scriptengine.java.JavaCompiledScript] = Box tryo { - val packageExp = UUID.randomUUID().toString.replaceAll("^|-", "_") - val packageMatcher = Pattern.compile("""(?m)^\s*package\s+\S+?\s*;""").matcher(methodBody) - - val javaCode = s"""package code.api.util.dynamic.${packageExp}; - |${packageMatcher.replaceFirst("")} - |""".stripMargin - - javaEngine.asInstanceOf[javax.script.Compilable].compile(javaCode) - .asInstanceOf[ch.obermuhlner.scriptengine.java.JavaCompiledScript] - } + // Only the compile step is memoized — deterministic given the same source string, and the + // one genuinely expensive part (a real javax.tools.JavaCompiler invocation). Dependency + // validation below is NOT memoized: it depends on mutable external config + // (dynamic_code_compile_validate_enable/_dependencies), which can change between two + // createJavaHttp4sEndpoint calls for the identical source string — e.g. a doc compiled once + // while validation was off, then a later create/update call resubmitting the exact same + // method_body after validation was turned on and the whitelist tightened. An earlier version + // of this function memoized the validated *result* (Box[Http4sEndpointIO]) as a single unit, + // so that second call silently reused the first call's unvalidated success — bypassing the + // now-stricter policy for any resubmitted source. Re-running validation on every call costs + // little: it is Javassist bytecode inspection plus a Map lookup, not another compile. + val compiledScriptBox: Box[ch.obermuhlner.scriptengine.java.JavaCompiledScript] = + memoJavaCompiledScript.memoize("java-http4s-endpoint:" + methodBody) { + // Real compile happens here (javax.tools.JavaCompiler via the JSR-223 "java" engine) — + // any Java syntax/type error surfaces as an exception, caught by this `Box tryo` and + // turned into a Failure. + Box tryo { + val packageExp = UUID.randomUUID().toString.replaceAll("^|-", "_") + val packageMatcher = Pattern.compile("""(?m)^\s*package\s+\S+?\s*;""").matcher(methodBody) + + val javaCode = s"""package code.api.util.dynamic.${packageExp}; + |${packageMatcher.replaceFirst("")} + |""".stripMargin + + javaEngine.asInstanceOf[javax.script.Compilable].compile(javaCode) + .asInstanceOf[ch.obermuhlner.scriptengine.java.JavaCompiledScript] + } + } - // Deliberately outside compiledScriptBox's `Box tryo`: a rejection here throws - // JsonResponseException, which must propagate UNCAUGHT (mirroring the Scala path's - // CompiledObjects.validateDependency(), also never wrapped in tryo) so + // Deliberately outside compiledScriptBox's `Box tryo` AND outside the memoization above: a + // rejection here throws JsonResponseException, which must propagate UNCAUGHT (mirroring the + // Scala path's CompiledObjects.validateDependency(), also never wrapped in tryo) so // compileDynamicResourceDoc's `case e: JsonResponseException => throw e` sees it intact. // JsonResponseException never sets a Throwable message (getMessage == null); Box.tryo would // catch it into Failure(null, Full(theException), Empty), and DynamicEndpoints.scala's @@ -667,7 +690,9 @@ object DynamicUtil extends MdcLoggable{ // what the Java code actually calls. Read the bytes directly from the classloader's private // byte map (reflection is unavoidable here: java-scriptengine exposes no public accessor) // and hand them to Javassist explicitly via ByteArrayClassPath, so the real method bodies — - // including any restricted OBP call — are visible to validation. + // including any restricted OBP call — are visible to validation. Re-registering the same + // bytes on a compile-cache hit is harmless: Javassist's ClassPool just gains a duplicate, + // already-matching ClassPath entry. val classBytesField = compiledClass.getClassLoader.getClass.getDeclaredField("mapClassBytes") classBytesField.setAccessible(true) val classBytes = classBytesField.get(compiledClass.getClassLoader) @@ -676,7 +701,8 @@ object DynamicUtil extends MdcLoggable{ .appendClassPath(new javassist.ByteArrayClassPath(compiledClass.getName, classBytes)) // Validate the real compiled Supplier class before it's ever invoked — see the doc comment - // above for why this must run against getCompiledInstance, not `func`/`this.partialFunction`. + // above for why this must run against getCompiledInstance, not `func`/`this.partialFunction`, + // and why it must run fresh on every call rather than being cached with the compile result. Validation.validateDependency(compiledScript.getCompiledInstance) val func = compiledScript.eval().asInstanceOf[java.util.function.Function[Array[AnyRef], Any]] diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala index e54d7a5f19..c7ceaae1ab 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala @@ -39,18 +39,24 @@ class DynamicResourceDocJavaSecurityValidationTest extends V400ServerSetup { |} |""".stripMargin + // Mirrors sample.props.template's default dynamic_code_compile_validate_dependencies exactly, + // minus the trailing newlines/line-continuations -- deliberately does NOT list APIUtil.getPropsValue. + private def defaultDependenciesWhitelist: String = + """[NewStyle.function.getClass.getTypeName -> "*", CompiledObjects.getClass.getTypeName -> "sandbox", HttpCode.getClass.getTypeName -> "200", DynamicCompileEndpoint.getClass.getTypeName -> "getPathParams, scalaFutureToBoxedJsonResponse", APIUtil.getClass.getTypeName -> "errorJsonResponse, errorJsonResponse$default$1, errorJsonResponse$default$2, errorJsonResponse$default$3, errorJsonResponse$default$4, scalaFutureToLaFuture, futureToBoxedResponse", ErrorMessages.getClass.getTypeName -> "*", ExecutionContext.Implicits.getClass.getTypeName -> "global", JSONFactory400.getClass.getTypeName -> "createBanksJson", classOf[Sandbox].getTypeName -> "runInSandbox", classOf[CallContext].getTypeName -> "*", classOf[ResourceDoc].getTypeName -> "getPathParams", "scala.reflect.runtime.package$" -> "universe", PractiseEndpoint.getClass.getTypeName + "*" -> "*"]""" + + private def enableStrictValidation(): Unit = setPropsValues( + "show_used_connector_methods" -> "true", + "dynamic_code_compile_validate_enable" -> "true", + "dynamic_code_compile_validate_dependencies" -> defaultDependenciesWhitelist + ) + + private def createRequest = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + feature("Security validation of Java method_body against dynamic_code_compile_validate_dependencies") { scenario("Registering a Java doc that calls a non-whitelisted OBP method is rejected with 400") { - setPropsValues( - "show_used_connector_methods" -> "true", - "dynamic_code_compile_validate_enable" -> "true", - "dynamic_code_compile_validate_dependencies" -> - """[NewStyle.function.getClass.getTypeName -> "*", CompiledObjects.getClass.getTypeName -> "sandbox", HttpCode.getClass.getTypeName -> "200", DynamicCompileEndpoint.getClass.getTypeName -> "getPathParams, scalaFutureToBoxedJsonResponse", APIUtil.getClass.getTypeName -> "errorJsonResponse, errorJsonResponse$default$1, errorJsonResponse$default$2, errorJsonResponse$default$3, errorJsonResponse$default$4, scalaFutureToLaFuture, futureToBoxedResponse", ErrorMessages.getClass.getTypeName -> "*", ExecutionContext.Implicits.getClass.getTypeName -> "global", JSONFactory400.getClass.getTypeName -> "createBanksJson", classOf[Sandbox].getTypeName -> "runInSandbox", classOf[CallContext].getTypeName -> "*", classOf[ResourceDoc].getTypeName -> "getPathParams", "scala.reflect.runtime.package$" -> "universe", PractiseEndpoint.getClass.getTypeName + "*" -> "*"]""" - ) - + enableStrictValidation() Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) - val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( dynamicResourceDocId = None, bankId = None, @@ -60,11 +66,48 @@ class DynamicResourceDocJavaSecurityValidationTest extends V400ServerSetup { methodBody = java.net.URLEncoder.encode(maliciousMethodBody, "UTF-8"), programmingLang = "Java" ) - val resp = makePostRequest(createReq, write(doc)) + val resp = makePostRequest(createRequest, write(doc)) Then("the compile is rejected with 400 DynamicResourceDocMethodDependency, not accepted") resp.code should equal(400) resp.body.extract[ErrorMessage].message should include(DynamicResourceDocMethodDependency) } + + // Regression guard for the bug createJavaHttp4sEndpoint had before it split compilation from + // validation: memoJavaCompiledScript (formerly memoJavaHttp4sEndpoint) memoized the WHOLE + // Box[Http4sEndpointIO], keyed only by the exact method_body string. A doc compiled once while + // validation was off got a cached Full(...) that a later, identical create call -- made AFTER + // validation was turned on and the whitelist tightened -- would silently reuse, never + // re-running Validation.validateDependency at all. This scenario reproduces exactly that + // sequence: compile the same malicious source once with validation off (succeeds, populates + // the compile cache), then enable strict validation and resubmit the identical source. + scenario("A Java source compiled once while validation was off is still validated on a later create call") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val firstDoc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "cacheBypassProbeTest1", + requestUrl = "/cache_bypass_probe_test_1/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(maliciousMethodBody, "UTF-8"), + programmingLang = "Java" + ) + When("validation is off (the suite default) and we compile the malicious source for the first time") + val firstResp = makePostRequest(createRequest, write(firstDoc)) + firstResp.code should equal(201) + + When("validation is then turned on with the same source resubmitted under a different doc") + enableStrictValidation() + val secondDoc = firstDoc.copy( + partialFunctionName = "cacheBypassProbeTest2", + requestUrl = "/cache_bypass_probe_test_2/MY_USER_ID" + ) + val secondResp = makePostRequest(createRequest, write(secondDoc)) + + Then("the second create is still rejected -- the compile-result cache must not bypass fresh validation") + secondResp.code should equal(400) + secondResp.body.extract[ErrorMessage].message should include(DynamicResourceDocMethodDependency) + } } } From 45a965a8f6c93c3fcce0800b8eeceb39fd714cb5 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 08:38:50 +0200 Subject: [PATCH 47/59] fix: restore Berlin Group precedence and stop registering an empty API version Two defects found reviewing the registry against the union it replaced. Berlin Group and UK Open Banking both publish getBalances, getAccountList and getAccountBalances. Http4s600's top-apis/popular-apis and JSONFactory6.0.0's metrics resolve a partialFunctionName with `.toMap`, which keeps the LAST matching entry, so registry order decides the operation_id they report. The hand-written union listed UK before BG, giving Berlin Group all three; sorting the scanned standards alphabetically put UK last and silently flipped them to UKv4.0.1-getBalances, UKv2.0-getAccountList and UKv2.0-getAccountBalances. Replace the alphabetical sort with an explicit standardPrecedence (UK Open Banking, then Berlin Group) and move Berlin Group v1.3 out of the explicit block so it is ordered by that precedence rather than pinned ahead of it. A standard absent from the list -- including the alias, whose apiStandard is whatever berlin_group_v1_3_alias_path names -- ranks below all of them and can never override a first-class standard. Verified against a running instance: the three names resolve to BGv1.3-getBalances, BGv2-getAccountList and BGv2-getAccountBalances again, matching the values measured before this branch. A configuration-gated standard that is switched off reports ScannedApiVersion("", "", ""), whose fullyQualifiedVersion is "" as well. While ScannedApis kept that registration, ApiVersionUtils.valueOf("") resolved successfully and, because the resource-docs route tolerates an empty path segment, GET /obp/v7.0.0/resource-docs//obp answered 200 with an empty document list where any other unknown version string gets 400 InvalidApiVersionString. Drop unaddressable registrations in ScannedApis.versionMapScannedApis, which fixes ApiVersionUtils, ResourceDocRegistry and Boot's version enablement in one place. Verified: that request now returns 400, and BG v1.3, BG v2, UK 4.0.1 and OBP v7.0.0 still serve 55, 22, 89 and 1031 docs. With the alias no longer registered while inactive it is not a registry surface at all, so the parity test's now-unreachable "cancel when unconfigured" branch is removed. Both defects reached a green CI because nothing asserted either value; two scenarios now pin them. Full local suite 3575/0. --- .../code/api/util/ResourceDocRegistry.scala | 64 +++++++++++-------- .../scala/code/api/util/ScannedApis.scala | 12 ++++ .../util/ResourceDocRegistryParityTest.scala | 48 +++++++++++--- 3 files changed, 89 insertions(+), 35 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala b/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala index 882c4683e3..88104642fb 100644 --- a/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala +++ b/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala @@ -40,7 +40,6 @@ object ResourceDocRegistry { lazy val registry: ListMap[ApiVersion, () => Seq[ResourceDoc]] = { val explicit: ListMap[ApiVersion, () => Seq[ResourceDoc]] = ListMap( v7_0_0 -> (() => code.api.v7_0_0.Http4s700.allResourceDocs.toSeq), - ConstantsBG.berlinGroupVersion1 -> (() => code.api.berlin.group.v1_3.Http4sBGv13.resourceDocs.toSeq), v6_0_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v600.toSeq), v5_1_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v510.toSeq), v5_0_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v500.toSeq), @@ -55,41 +54,56 @@ object ResourceDocRegistry { v1_2_1 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v121.toSeq), `dynamic-endpoint` -> (() => code.api.dynamic.endpoint.OBPAPIDynamicEndpoint.allResourceDocs.toSeq), `dynamic-entity` -> (() => code.api.dynamic.entity.OBPAPIDynamicEntity.allResourceDocs.toSeq) - // Berlin Group v2 is NOT listed here -- Http4sBGv2 is a ScannedApis registrant (its - // apiVersion is ConstantsBG.berlinGroupVersion2), so it is picked up by `scanned` below. + // Neither Berlin Group nor UK Open Banking is listed here: they are all ScannedApis + // registrants, so `scanned` picks them up and -- crucially -- orders them against each other + // by standardPrecedence below. Naming one of them here would pin it ahead of that ordering. ) // Every standard discovered via ScannedApis (UK OB 200/310/401, BG v1.3 canonical + alias, - // BG v2, and any future `with ScannedApis` standard). Explicit entries win on key collision -- - // BG v1.3 canonical is both explicit above and a registrant, same underlying buffer either way. + // BG v2, and any future `with ScannedApis` standard), folded into a ListMap so the registry has + // ONE defined iteration order. // - // Sorted, and folded into a ListMap, so the registry has ONE defined iteration order. - // ScannedApis.versionMapScannedApis is an unordered Map, and several standards share - // partialFunctionNames (the BG v1.3 alias re-stamps the canonical BG v1.3 docs, so it collides - // with both BG v1.3 and -- on getAccountDetails, getAccountList, getCardAccountBalances, - // getCardAccountTransactionList, getTransactionDetails -- with BG v2). Consumers such as - // Http4s600's top-apis/popular-apis and JSONFactory6.0.0's metrics build - // `partialFunctionName -> operationId` with `.toMap`, where the LAST entry wins, so leaving the - // order to Map's hash iteration would leave the reported operation_id undefined and let it - // shift silently whenever a standard is added or removed. - // - // The sort key is (apiStandard, apiShortVersion) rather than fullyQualifiedVersion because it - // cannot tie: that pair is exactly ScannedApiVersion's equals/hashCode key, so two distinct - // keys of this Map always differ in it, and sortBy therefore yields a total order rather than - // falling back to the unordered input for ties. fullyQualifiedVersion concatenates the two - // (apiStandard.toUpperCase + apiShortVersion) and so can collide across distinct keys -- - // ("BG", "v1.3") and ("BGV", "1.3") both render "BGV1.3" -- which a deployment could reach by - // configuring berlin_group_v1_3_alias_path. The resulting order is unchanged in practice: - // alias ("0.6"/"") first, then BG v1.3, BG v2, then UK 2.0/3.1/4.0.1, so BG v2 keeps winning - // the names it shares with the alias, matching the behaviour before this branch. + // Order matters beyond determinism: Http4s600's top-apis/popular-apis and JSONFactory6.0.0's + // metrics build `partialFunctionName -> operationId` with `.toMap`, where the LAST entry wins. + // Berlin Group and UK Open Banking share three partialFunctionNames -- getBalances, + // getAccountList, getAccountBalances -- and the hand-written union this registry replaced + // listed UK before BG, so Berlin Group won all three. Sorting alphabetically put UK last and + // silently flipped them to UKv4.0.1-getBalances / UKv2.0-getAccountList / + // UKv2.0-getAccountBalances in metrics output, so the precedence is now explicit. val scanned: ListMap[ApiVersion, () => Seq[ResourceDoc]] = ScannedApis.versionMapScannedApis.toSeq .collect { case (version: ScannedApiVersion, apis) if !explicit.contains(version) => version -> (() => apis.allResourceDocs.toSeq) } - .sortBy(entry => (entry._1.apiStandard, entry._1.apiShortVersion)) + .sortBy(entry => sortKey(entry._1)) .foldLeft(ListMap.empty[ApiVersion, () => Seq[ResourceDoc]])(_ + _) explicit ++ scanned } + /** + * Standards in ASCENDING precedence: a standard later in this list wins a partialFunctionName it + * shares with an earlier one, because the `.toMap` consumers keep the last entry. This + * reproduces the order of the hand-written union that preceded this registry (UK Open Banking, + * then Berlin Group). + * + * A standard that is not listed ranks below all of them -- including the Berlin Group v1.3 alias, + * whose apiStandard is whatever `berlin_group_v1_3_alias_path` names, so it can never override a + * first-class standard no matter how a deployment configures it. + */ + private val standardPrecedence: List[String] = + List(ApiVersion.ukOpenBankingV20.apiStandard, ConstantsBG.berlinGroupVersion1.apiStandard) + + /** + * Total order over registry keys: precedence first, then the version's own identity. + * + * The tie-breaker is (apiStandard, apiShortVersion) rather than fullyQualifiedVersion because + * that pair is exactly ScannedApiVersion's equals/hashCode key, so two distinct keys of a Map + * keyed by version always differ in it and sortBy never has to fall back to the unordered input. + * fullyQualifiedVersion concatenates the two (apiStandard.toUpperCase + apiShortVersion) and can + * therefore collide across distinct keys -- ("BG", "v1.3") and ("BGV", "1.3") both render + * "BGV1.3" -- which a deployment could reach through berlin_group_v1_3_alias_path. + */ + private def sortKey(version: ScannedApiVersion): (Int, String, String) = + (standardPrecedence.indexOf(version.apiStandard), version.apiStandard, version.apiShortVersion) + /** What the per-version resource-docs dispatcher serves for this version (empty if unknown). */ def docsFor(version: ApiVersion): Seq[ResourceDoc] = registry.get(version).map(_ ()).getOrElse(Nil) diff --git a/obp-api/src/main/scala/code/api/util/ScannedApis.scala b/obp-api/src/main/scala/code/api/util/ScannedApis.scala index bfbc688985..238de80b0e 100644 --- a/obp-api/src/main/scala/code/api/util/ScannedApis.scala +++ b/obp-api/src/main/scala/code/api/util/ScannedApis.scala @@ -21,9 +21,21 @@ trait ScannedApis { object ScannedApis { /** * this map value are all scanned objects those extends ScannedApiVersion, the key is it apiVersion field + * + * Registrants whose version carries no urlPrefix, apiStandard and apiShortVersion are dropped: + * such a version addresses nothing, and it is how a configuration-gated standard reports itself + * as switched off (OBP_BERLIN_GROUP_1_3_Alias falls back to ScannedApiVersion("", "", "") when + * berlin_group_v1_3_alias_path is unset). Keeping it here leaked into everything built from this + * map: its fullyQualifiedVersion is "" too, so ApiVersionUtils.valueOf("") resolved successfully + * and GET /obp/v7.0.0/resource-docs//obp answered 200 with an empty document list instead of the + * 400 InvalidApiVersionString any other unknown version string gets. */ lazy val versionMapScannedApis: Map[ScannedApiVersion, ScannedApis] = ClassScanUtils.getSubTypeObjects[ScannedApis] + .filter(it => isAddressable(it.apiVersion)) .map(it=> (it.apiVersion, it)) .toMap + + private def isAddressable(version: ScannedApiVersion): Boolean = + version.urlPrefix.trim.nonEmpty || version.apiStandard.trim.nonEmpty || version.apiShortVersion.trim.nonEmpty } diff --git a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala index 25b065255b..cbb5f9c624 100644 --- a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala +++ b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala @@ -1,6 +1,6 @@ package code.api.util -import code.api.berlin.group.v1_3.{Http4sBGv13Alias, OBP_BERLIN_GROUP_1_3_Alias} +import code.api.berlin.group.v1_3.Http4sBGv13Alias import code.setup.ServerSetup import com.openbankproject.commons.util.{ApiStandards, ApiVersion, ScannedApiVersion} import org.scalatest.Tag @@ -26,19 +26,19 @@ class ResourceDocRegistryParityTest extends ServerSetup { private lazy val allOperationIds: Set[String] = APIUtil.getAllResourceDocs.map(_.operationId).toSet - // The Berlin Group v1.3 alias is the one surface in the union a deployment can switch off: + // The Berlin Group v1.3 alias is the one surface a deployment can switch off: // berlin_group_v1_3_alias_path is unset by default and is supplied for test runs by // test.default.props and by the two CI workflows. test.default.props is gitignored // (.gitignore:21), so a fresh clone, a colleague's checkout or an IDE ScalaTest run may not have - // it -- assertions that genuinely need the alias cancel there instead of failing with a message - // that gives no hint a prop is missing. - private lazy val aliasVersion: ApiVersion = OBP_BERLIN_GROUP_1_3_Alias.apiVersion + // it -- the pin below cancels there instead of failing with a message that gives no hint a prop + // is missing. The per-surface loop needs no such guard: an unconfigured alias reports the + // unaddressable ScannedApiVersion("", "", ""), which ScannedApis now drops, so it is not a + // surface at all rather than an empty one. private lazy val aliasIsConfigured: Boolean = Http4sBGv13Alias.resourceDocs.nonEmpty private val aliasNotConfigured = "berlin_group_v1_3_alias_path is not set, so the Berlin Group v1.3 alias contributes no docs" private def label(version: ApiVersion): String = version match { - case v if v == aliasVersion && !aliasIsConfigured => "Berlin Group v1.3 alias (not configured)" case sv: ScannedApiVersion => sv.fullyQualifiedVersion case other => other.toString } @@ -48,18 +48,17 @@ class ResourceDocRegistryParityTest extends ServerSetup { // deliberately out of the union; see ResourceDocRegistry.obpUnionVersion for why, and for the // accepted consequence that an operation id living only in a superseded aggregation stays // unresolvable. - private lazy val surfaces: List[(ApiVersion, String, Seq[String])] = + private lazy val surfaces: List[(String, Seq[String])] = ResourceDocRegistry.unionVersions.toList - .map(version => (version, label(version), ResourceDocRegistry.docsFor(version).map(_.operationId))) + .map(version => (label(version), ResourceDocRegistry.docsFor(version).map(_.operationId))) feature("getAllResourceDocs contains every per-standard resource-doc surface the union covers") { scenario("the registry itself is non-empty", RegistryParityTag) { surfaces should not be empty } - surfaces.foreach { case (version, label, operationIds) => + surfaces.foreach { case (label, operationIds) => scenario(s"$label operation ids are all resolvable globally", RegistryParityTag) { - if (version == aliasVersion && !aliasIsConfigured) cancel(aliasNotConfigured) // Non-empty matters as much as membership: an empty surface is trivially a subset of the // union, so without this a standard whose docs silently stop being registered (the very // failure mode this test exists for) would pass unnoticed. @@ -103,6 +102,35 @@ class ResourceDocRegistryParityTest extends ServerSetup { } } + // Berlin Group and UK Open Banking both publish getBalances, getAccountList and + // getAccountBalances. Http4s600's top-apis/popular-apis and JSONFactory6.0.0's metrics resolve + // a partialFunctionName with `.toMap`, which keeps the LAST matching entry, so the registry's + // iteration order decides the operation_id those endpoints report. The hand-written union that + // preceded this registry listed UK before BG, giving Berlin Group the three names; sorting the + // scanned standards alphabetically silently handed them to UK Open Banking instead. This pins + // the resolved values so the precedence cannot drift again unnoticed. + scenario("Berlin Group keeps the partialFunctionNames it shares with UK Open Banking", RegistryParityTag) { + val resolved = APIUtil.getAllResourceDocs + .map(doc => doc.partialFunctionName -> doc.operationId).toMap + resolved.get("getBalances") shouldBe Some("BGv1.3-getBalances") + resolved.get("getAccountList") shouldBe Some("BGv2-getAccountList") + resolved.get("getAccountBalances") shouldBe Some("BGv2-getAccountBalances") + } + + // An unconfigured configuration-gated standard reports ScannedApiVersion("", "", ""), whose + // fullyQualifiedVersion is "" as well. While ScannedApis kept it, ApiVersionUtils.valueOf("") + // resolved successfully and GET /obp/v7.0.0/resource-docs//obp answered 200 with an empty + // document list instead of the 400 every other unknown version string gets. + scenario("an unaddressable empty version is not a registered API version", RegistryParityTag) { + ScannedApis.versionMapScannedApis.keys.foreach { version => + withClue(s"$version was registered despite addressing nothing ") { + (version.urlPrefix.trim + version.apiStandard.trim + version.apiShortVersion.trim) should not be empty + } + } + ApiVersionUtils.versions.map(_.fullyQualifiedVersion) should not contain "" + an[IllegalArgumentException] should be thrownBy ApiVersionUtils.valueOf("") + } + // The three named pins below are the three historical drift instances. They are NOT redundant // with the loop above: both sides of that loop are now derived from ResourceDocRegistry, so its // membership half holds by construction and cannot fail. What the loop still catches is a From a369957d8bcfded5a365cdfa3af5ea074f18ec5e Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 08:41:03 +0200 Subject: [PATCH 48/59] fix: satisfy CI's test-isolation lint for the setPropsValues helper enableStrictValidation() wrapped setPropsValues with an expression body (def ... = setPropsValues(...)), which .github/scripts/ check_test_isolation.py's brace-based scanner does not recognize as a safe "helper" scope -- it only classifies a def as safe when it finds an opening brace immediately following the def name. The call was flagged as running at class-instantiation time and failed CI's lint step before compilation even started. Verified locally with python3 .github/scripts/check_test_isolation.py. --- ...micResourceDocJavaSecurityValidationTest.scala | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala index c7ceaae1ab..3e056158c7 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala @@ -44,11 +44,16 @@ class DynamicResourceDocJavaSecurityValidationTest extends V400ServerSetup { private def defaultDependenciesWhitelist: String = """[NewStyle.function.getClass.getTypeName -> "*", CompiledObjects.getClass.getTypeName -> "sandbox", HttpCode.getClass.getTypeName -> "200", DynamicCompileEndpoint.getClass.getTypeName -> "getPathParams, scalaFutureToBoxedJsonResponse", APIUtil.getClass.getTypeName -> "errorJsonResponse, errorJsonResponse$default$1, errorJsonResponse$default$2, errorJsonResponse$default$3, errorJsonResponse$default$4, scalaFutureToLaFuture, futureToBoxedResponse", ErrorMessages.getClass.getTypeName -> "*", ExecutionContext.Implicits.getClass.getTypeName -> "global", JSONFactory400.getClass.getTypeName -> "createBanksJson", classOf[Sandbox].getTypeName -> "runInSandbox", classOf[CallContext].getTypeName -> "*", classOf[ResourceDoc].getTypeName -> "getPathParams", "scala.reflect.runtime.package$" -> "universe", PractiseEndpoint.getClass.getTypeName + "*" -> "*"]""" - private def enableStrictValidation(): Unit = setPropsValues( - "show_used_connector_methods" -> "true", - "dynamic_code_compile_validate_enable" -> "true", - "dynamic_code_compile_validate_dependencies" -> defaultDependenciesWhitelist - ) + // Block body (not `= setPropsValues(...)`) so .github/scripts/check_test_isolation.py's brace + // scanner sees an opening `{` right after `def enableStrictValidation` and treats this as a + // safe "helper called from scenarios" scope rather than a class-body-level setPropsValues call. + private def enableStrictValidation(): Unit = { + setPropsValues( + "show_used_connector_methods" -> "true", + "dynamic_code_compile_validate_enable" -> "true", + "dynamic_code_compile_validate_dependencies" -> defaultDependenciesWhitelist + ) + } private def createRequest = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) From da5a64be9cb27168384e98f596dc23500e056b0a Mon Sep 17 00:00:00 2001 From: simonredfern Date: Tue, 1 Sep 2026 08:49:07 +0200 Subject: [PATCH 49/59] Fix to Account Application v3.1.0 --- .../scala/code/api/v3_1_0/Http4s310.scala | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala b/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala index c6c8b4e74d..12ec0a2e1e 100644 --- a/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala +++ b/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala @@ -3063,8 +3063,9 @@ object Http4s310 { ) // ─── updateAccountApplicationStatus (PUT) ──────────────────────────────── - // Side effect: when status == "ACCEPTED", a new bank account is created for the - // logged-in user. Preserved verbatim from the Lift implementation. + // Side effect: when status == "ACCEPTED", a new bank account is created and the + // APPLICANT (the application's user) becomes its holder. The Lift implementation + // (and its verbatim port) made the logged-in approver the holder — fixed 2026-09. val updateAccountApplicationStatus: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ PUT -> `prefixPath` / "banks" / _ / "account-applications" / accountApplicationIdStr => @@ -3074,24 +3075,36 @@ object Http4s310 { _ <- NewStyle.function.tryons(s"$InvalidJsonFormat status should not be blank.", 400, Some(cc)) { org.apache.commons.lang3.Validate.notBlank(putJson.status) } - (_, _) <- NewStyle.function.getAccountApplicationById(accountApplicationIdStr, Some(cc)) - (accountApplication, _) <- NewStyle.function.updateAccountApplicationStatus(accountApplicationIdStr, putJson.status, Some(cc)) - userIdOpt = Option(accountApplication.userId) - customerIdOpt = Option(accountApplication.customerId) + (applicationBefore, _) <- NewStyle.function.getAccountApplicationById(accountApplicationIdStr, Some(cc)) + userIdOpt = Option(applicationBefore.userId) + customerIdOpt = Option(applicationBefore.customerId) appUser <- unboxOptionOBPReturnType(userIdOpt.map(NewStyle.function.findByUserId(_, Some(cc)))) customer <- unboxOptionOBPReturnType(customerIdOpt.map(NewStyle.function.getCustomerByCustomerId(_, Some(cc)))) + // Guard BEFORE the status transition commits: failing after it would strand the + // application as ACCEPTED with no account. A consent-user applicant can only come + // from a row that predates the creation-side guard (or was written another way). + _ <- code.util.Helper.booleanToFuture( + s"$InvalidUserId The application's user is a consent user (an agent identity minted by a Consent). Accounts are held by humans - re-apply with the granting user's USER_ID.", + failCode = 400, cc = Some(cc))(!appUser.exists(_.isConsentUser)) + (accountApplication, _) <- NewStyle.function.updateAccountApplicationStatus(accountApplicationIdStr, putJson.status, Some(cc)) _ <- putJson.status match { case "ACCEPTED" => + // The APPLICANT becomes the holder. The Lift-era code (ported verbatim) made the + // approving admin the holder and left appUser unused — every accepted application + // handed the account to whoever clicked approve. Customer-only applications + // (userId empty) keep the legacy approver-as-holder behaviour: there is no user + // to hold, and refusing here would strand the just-committed ACCEPTED status. for { accountId <- Future(AccountId(java.util.UUID.randomUUID().toString)) + holder = appUser.getOrElse(user) (_, _) <- NewStyle.function.createBankAccount( bank.bankId, accountId, accountApplication.productCode.value, "", "EUR", BigDecimal("0"), - user.name, "", + holder.name, "", List.empty, Some(cc)) success <- code.model.dataAccess.BankAccountCreation.setAccountHolderAndRefreshUserAccountAccess( - bank.bankId, accountId, user, Some(cc)) + bank.bankId, accountId, holder, Some(cc)) } yield success case _ => Future("") } @@ -3227,6 +3240,12 @@ object Http4s310 { org.apache.commons.lang3.Validate.isTrue(postedData.user_id.isDefined || postedData.customer_id.isDefined) } appUser <- unboxOptionOBPReturnType(postedData.user_id.map(NewStyle.function.findByUserId(_, Some(cc)))) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + // On ACCEPTED the application's user becomes the account holder, so a consent + // user must be rejected here, before the application is stored. + _ <- code.util.Helper.booleanToFuture( + s"$InvalidUserId user_id names a consent user (an agent identity minted by a Consent). Accounts are held by humans - use the granting user's USER_ID.", + failCode = 400, cc = Some(cc))(!appUser.exists(_.isConsentUser)) customer <- unboxOptionOBPReturnType(postedData.customer_id.map(NewStyle.function.getCustomerByCustomerId(_, Some(cc)))) (accountApplication, _) <- NewStyle.function.createAccountApplication( productCode = ProductCode(postedData.product_code), From b7bef716036d7f241006aed03e7d8c0b2a836bdf Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 09:09:12 +0200 Subject: [PATCH 50/59] fix: decouple dependency-whitelist validation from show_used_connector_methods getDynamicCodeDependentMethods and APIUtil.getDependentMethods were both gated by SHOW_USED_CONNECTOR_METHODS, an unrelated introspection/reporting toggle that defaults to false. This meant dynamic_code_compile_validate_enable alone did nothing: with the reporting prop left at its default, the bytecode scan always returned an empty dependency list, so every dynamic-code call passed the whitelist check regardless of what it actually invoked. Add a force parameter that Validation.validateDependency sets to true, making security validation controlled solely by dynamic_code_compile_validate_enable as documented. The existing caller in DynamicCompileEndpoint keeps its prior behaviour unchanged. --- .../main/scala/code/api/util/APIUtil.scala | 6 +++-- .../scala/code/api/util/DynamicUtil.scala | 27 ++++++++++++++----- ...esourceDocJavaSecurityValidationTest.scala | 8 +++++- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index 9e45f49432..e591a362bd 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -4399,8 +4399,10 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ * * than the return value may be (getUserAndSessionContextFuture, ***,***),(map,***,***), (getOrElse,***,***) ...... */ - def getDependentMethods(className: String, methodName:String, signature: String): List[(String, String, String)] = { - if (SHOW_USED_CONNECTOR_METHODS) { + // force bypasses the SHOW_USED_CONNECTOR_METHODS gate below -- see + // DynamicUtil.getDynamicCodeDependentMethods' doc comment for why security validation needs this. + def getDependentMethods(className: String, methodName:String, signature: String, force: Boolean = false): List[(String, String, String)] = { + if (SHOW_USED_CONNECTOR_METHODS || force) { val methods = ListBuffer[(String, String, String)]() //NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. //eg: className == code.api.UKOpenBanking.v3_1_0.APIMethods_AccountAccessApi$$anonfun$createAccountAccessConsents$lzycompute$1 diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 6c37de021b..173bdca707 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -170,26 +170,37 @@ object DynamicUtil extends MdcLoggable{ } /** - * NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. + * NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. * @param clazz * @param predicate + * @param force bypasses the SHOW_USED_CONNECTOR_METHODS gate below. SHOW_USED_CONNECTOR_METHODS + * exists to opt in to an unrelated, expensive introspection/reporting feature (which + * connector methods a static endpoint touches) — it was never meant to gate SECURITY + * validation, which reuses this same bytecode scan. Without `force`, a deployment + * that sets dynamic_code_compile_validate_enable=true (the documented, security- + * relevant prop) but leaves the unrelated show_used_connector_methods at its default + * false would silently get an always-empty dependency list here — every dynamic-code + * call looks "allowed" no matter what it does, because there is nothing to check + * against the whitelist. Validation.validateDependency passes force=true so it is + * controlled solely by dynamic_code_compile_validate_enable, matching what an + * operator following that prop's own documentation would expect. * @return */ - def getDynamicCodeDependentMethods(clazz: Class[_], predicate: String => Boolean = _ => true): List[(String, String, String)] = - if (SHOW_USED_CONNECTOR_METHODS) { + def getDynamicCodeDependentMethods(clazz: Class[_], predicate: String => Boolean = _ => true, force: Boolean = false): List[(String, String, String)] = + if (SHOW_USED_CONNECTOR_METHODS || force) { val className = clazz.getTypeName val listBuffer = new ListBuffer[(String, String, String)]() val classPool = getClassPool(clazz.getClassLoader) - //NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. + //NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. val ctClass = classPool.get(className) for { method <- ctClass.getDeclaredMethods.toList if predicate(method.getName) - ternary @ (typeName, methodName, signature) <- APIUtil.getDependentMethods(className, method.getName, method.getSignature) + ternary @ (typeName, methodName, signature) <- APIUtil.getDependentMethods(className, method.getName, method.getSignature, force) } yield { // if method is also dynamic compile code, extract it's dependent method if(className.startsWith(typeName) && methodName.startsWith(clazz.getPackage.getName+ "$")) { - listBuffer.appendAll(APIUtil.getDependentMethods(typeName, methodName, signature)) + listBuffer.appendAll(APIUtil.getDependentMethods(typeName, methodName, signature, force)) } else { listBuffer.append(ternary) } @@ -478,7 +489,9 @@ object DynamicUtil extends MdcLoggable{ def validateDependency(obj: AnyRef): Unit = { if(APIUtil.getPropsAsBoolValue("dynamic_code_compile_validate_enable",false)){ - val dependentMethods: List[(String, String, String)] = DynamicUtil.getDynamicCodeDependentMethods(obj.getClass) + // force=true: this check must not also require the unrelated show_used_connector_methods + // prop -- see getDynamicCodeDependentMethods' doc comment for why. + val dependentMethods: List[(String, String, String)] = DynamicUtil.getDynamicCodeDependentMethods(obj.getClass, force = true) validateDependency(dependentMethods) } else{ // If false, nothing to do here. ; diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala index 3e056158c7..66fabae06b 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala @@ -44,12 +44,18 @@ class DynamicResourceDocJavaSecurityValidationTest extends V400ServerSetup { private def defaultDependenciesWhitelist: String = """[NewStyle.function.getClass.getTypeName -> "*", CompiledObjects.getClass.getTypeName -> "sandbox", HttpCode.getClass.getTypeName -> "200", DynamicCompileEndpoint.getClass.getTypeName -> "getPathParams, scalaFutureToBoxedJsonResponse", APIUtil.getClass.getTypeName -> "errorJsonResponse, errorJsonResponse$default$1, errorJsonResponse$default$2, errorJsonResponse$default$3, errorJsonResponse$default$4, scalaFutureToLaFuture, futureToBoxedResponse", ErrorMessages.getClass.getTypeName -> "*", ExecutionContext.Implicits.getClass.getTypeName -> "global", JSONFactory400.getClass.getTypeName -> "createBanksJson", classOf[Sandbox].getTypeName -> "runInSandbox", classOf[CallContext].getTypeName -> "*", classOf[ResourceDoc].getTypeName -> "getPathParams", "scala.reflect.runtime.package$" -> "universe", PractiseEndpoint.getClass.getTypeName + "*" -> "*"]""" + // Deliberately does NOT set show_used_connector_methods: that prop exists to opt in to an + // unrelated, expensive introspection/reporting feature and was never meant to gate security + // validation, which happens to reuse the same underlying bytecode scan. An operator who reads + // only dynamic_code_compile_validate_enable's own prop documentation and sets just these two + // props (as this method does) must still get real enforcement -- proving that is the point of + // every scenario below. + // // Block body (not `= setPropsValues(...)`) so .github/scripts/check_test_isolation.py's brace // scanner sees an opening `{` right after `def enableStrictValidation` and treats this as a // safe "helper called from scenarios" scope rather than a class-body-level setPropsValues call. private def enableStrictValidation(): Unit = { setPropsValues( - "show_used_connector_methods" -> "true", "dynamic_code_compile_validate_enable" -> "true", "dynamic_code_compile_validate_dependencies" -> defaultDependenciesWhitelist ) From eee46acd848ab3574ecce0d942ac2bc99d81c297 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 09:10:09 +0200 Subject: [PATCH 51/59] fix: register compiled Java bytecode with ClassPool only once per source createJavaHttp4sEndpoint re-registered the compiled class's bytes into the shared Javassist ClassPool on every call, including compile-cache hits. ClassPool.appendClassPath has no dedup, so a long-running process serving the same dynamic endpoint repeatedly (e.g. on each resourceDocs-list rebuild) grew the ClassPool's classpath chain without bound. Move the registration inside the compile memoization block so it runs exactly once per distinct source string, tied to the same cache lifetime as the compiled class itself. --- .../scala/code/api/util/DynamicUtil.scala | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 173bdca707..fee661611b 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -678,8 +678,32 @@ object DynamicUtil extends MdcLoggable{ |${packageMatcher.replaceFirst("")} |""".stripMargin - javaEngine.asInstanceOf[javax.script.Compilable].compile(javaCode) + val compiledScript = javaEngine.asInstanceOf[javax.script.Compilable].compile(javaCode) .asInstanceOf[ch.obermuhlner.scriptengine.java.JavaCompiledScript] + + // getDynamicCodeDependentMethods loads a class's bytecode via Javassist's + // LoaderClassPath, which reads it through classLoader.getResourceAsStream(...). The + // compiler's ch.obermuhlner.scriptengine.java.MemoryClassLoader only overrides + // loadClass() — it never exposes the compiled bytes as a classpath resource — so that + // lookup silently fails (javassist.NotFoundException) and validation would see zero + // dependent methods no matter what the Java code actually calls. Read the bytes + // directly from the classloader's private byte map (reflection is unavoidable here: + // java-scriptengine exposes no public accessor) and hand them to Javassist explicitly + // via ByteArrayClassPath, so the real method bodies — including any restricted OBP + // call — are visible to validation. Done here, inside the compile memoization, so it + // runs exactly once per distinct source: ClassPool.appendClassPath has no dedup of its + // own, so doing this on every createJavaHttp4sEndpoint call (as an earlier version of + // this function did, on every resourceDocs-list rebuild for the process's lifetime) + // grew that ClassPool's classpath chain without bound. + val compiledClass = compiledScript.getCompiledClass + val classBytesField = compiledClass.getClassLoader.getClass.getDeclaredField("mapClassBytes") + classBytesField.setAccessible(true) + val classBytes = classBytesField.get(compiledClass.getClassLoader) + .asInstanceOf[java.util.Map[String, Array[Byte]]].get(compiledClass.getName) + getClassPool(compiledClass.getClassLoader) + .appendClassPath(new javassist.ByteArrayClassPath(compiledClass.getName, classBytes)) + + compiledScript } } @@ -693,26 +717,6 @@ object DynamicUtil extends MdcLoggable{ // through to "compiled code return nothing" and discarding the real rejection reason. `.map` // does not swallow exceptions the way `Box tryo` does, so this stays uncaught here. compiledScriptBox.map { compiledScript => - val compiledClass = compiledScript.getCompiledClass - - // getDynamicCodeDependentMethods loads the class's bytecode via Javassist's LoaderClassPath, - // which reads it through classLoader.getResourceAsStream(...). The compiler's - // ch.obermuhlner.scriptengine.java.MemoryClassLoader only overrides loadClass() — it never - // exposes the compiled bytes as a classpath resource — so that lookup silently fails - // (javassist.NotFoundException) and validation would see zero dependent methods no matter - // what the Java code actually calls. Read the bytes directly from the classloader's private - // byte map (reflection is unavoidable here: java-scriptengine exposes no public accessor) - // and hand them to Javassist explicitly via ByteArrayClassPath, so the real method bodies — - // including any restricted OBP call — are visible to validation. Re-registering the same - // bytes on a compile-cache hit is harmless: Javassist's ClassPool just gains a duplicate, - // already-matching ClassPath entry. - val classBytesField = compiledClass.getClassLoader.getClass.getDeclaredField("mapClassBytes") - classBytesField.setAccessible(true) - val classBytes = classBytesField.get(compiledClass.getClassLoader) - .asInstanceOf[java.util.Map[String, Array[Byte]]].get(compiledClass.getName) - getClassPool(compiledClass.getClassLoader) - .appendClassPath(new javassist.ByteArrayClassPath(compiledClass.getName, classBytes)) - // Validate the real compiled Supplier class before it's ever invoked — see the doc comment // above for why this must run against getCompiledInstance, not `func`/`this.partialFunction`, // and why it must run fresh on every call rather than being cached with the compile result. From 04b1417ada0d964830c310c1a09dfdf65bab0b58 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 09:11:59 +0200 Subject: [PATCH 52/59] fix: fall back to Scala for dynamic resource docs with a NULL lang column getJsonDynamicResourceDoc passed Lang.get straight through as a constructor argument. Rows created before the Lang column existed have a genuine SQL NULL there, and an explicit null argument bypasses JsonDynamicResourceDoc's own "Scala" default -- that default only applies when the argument is omitted. Legacy rows therefore reported programming_lang as null/empty instead of the documented default. Fall back explicitly with Option(...).filter(isNotBlank).getOrElse("Scala"), matching the pattern already used for the other nullable text fields in the same method. Added a regression test that forces a genuine NULL via raw SQL (bypassing the ORM, which always writes "Scala") and asserts GET still reports "Scala". --- .../DynamicResourceDoc.scala | 6 +++- .../api/v4_0_0/DynamicResourceDocTest.scala | 36 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala index b46485383e..d42ef429fd 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala @@ -54,7 +54,11 @@ object DynamicResourceDoc extends DynamicResourceDoc with LongKeyedMetaMapper[Dy errorResponseBodies = dynamicResourceDoc.ErrorResponseBodies.get, tags = dynamicResourceDoc.Tags.get, roles = dynamicResourceDoc.Roles.get, - programmingLang = dynamicResourceDoc.Lang.get + // Rows created before the Lang column existed have NULL there, not "Scala" -- a bare + // Lang.get would surface that as an empty/null programming_lang instead of falling back to + // JsonDynamicResourceDoc's own "Scala" default, since an explicit null argument bypasses a + // case class default (that only applies when the argument is omitted entirely). + programmingLang = Option(dynamicResourceDoc.Lang.get).filter(StringUtils.isNotBlank).getOrElse("Scala") ) } diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala index ec60ccb3b9..c7d21954af 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala @@ -414,6 +414,42 @@ class DynamicResourceDocTest extends V400ServerSetup { storedRow.UpdatedByUserId.get should be(resourceUser1.userId) storedRow.MethodBodyHash.get should be(code.api.util.APIUtil.sha256Hex(URLDecoder.decode(changedMethodBody, "UTF-8"))) } + + // Regression guard: rows created before the Lang column existed have a genuine SQL NULL there + // (Schemifier's ALTER TABLE ADD COLUMN sets no default), not "Scala". An explicit null argument + // bypasses JsonDynamicResourceDoc's own programmingLang="Scala" default -- that default only + // applies when the argument is omitted entirely -- so a bare Lang.get would have surfaced as a + // null/empty programming_lang in the API response instead of falling back to "Scala". + scenario("A dynamic resource doc row predating the programming_lang column still reports \"Scala\"", ApiEndpoint1, ApiEndpoint3, VersionOfApi) { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canGetDynamicResourceDoc.toString) + + When("We create a dynamic resource doc") + val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val posted = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + partialFunctionName = "preDatesLangColumnTest", + requestUrl = "/pre_dates_lang_column_test/MY_USER_ID" + ) + val createResp = makePostRequest(createReq, write(posted)) + createResp.code should equal(201) + val docId = (createResp.body \ "dynamic_resource_doc_id").values.toString + + When("its lang column is forced to a genuine SQL NULL, bypassing the ORM (which always writes \"Scala\")") + import code.dynamicResourceDoc.DynamicResourceDoc + net.liftweb.mapper.DB.runUpdate( + s"UPDATE ${DynamicResourceDoc.dbTableName} SET ${DynamicResourceDoc.Lang.dbColumnName} = NULL " + + s"WHERE ${DynamicResourceDoc.DynamicResourceDocId.dbColumnName} = ?", + List(docId) + ) + + Then("GET still reports programming_lang as \"Scala\", not null or empty") + val getReq = (v4_0_0_Request / "management" / "dynamic-resource-docs" / docId).GET <@ (user1) + val getResp = makeGetRequest(getReq) + getResp.code should equal(200) + getResp.body.extract[JsonDynamicResourceDoc].programmingLang should equal("Scala") + } } } From 9b238b0d2e636b5e4d9d2fc9a172df2299d709db Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 10:14:36 +0200 Subject: [PATCH 53/59] fix: rank the Berlin Group v1.3 alias by identity, not by its configured name standardPrecedence ranked a version by its apiStandard string, and the alias takes that string from the first segment of berlin_group_v1_3_alias_path. A deployment may point it at a name an existing standard already uses: configured as "BG/v9" the alias reports ScannedApiVersion("BG", "BG", "v9"), ranks alongside Berlin Group, and -- sorting after "v2" on the tie-breaker -- comes last, so its re-stamped copies won getBalances, getAccountList and getAccountBalances away from the canonical docs it had copied. Metrics, top-apis and popular-apis would then report BGv9-getBalances instead of BGv1.3-getBalances. The comment on standardPrecedence claimed the opposite, that the alias "can never override a first-class standard no matter how a deployment configures it". Match the alias by identity instead and rank it below every listed standard, which makes that claim true for any configuration. sortKey takes the derived alias version as a curried parameter and is package-private so the guarantee can be tested against a synthetic alias, rather than only under whichever berlin_group_v1_3_alias_path the JVM happens to have booted with. Verified both directions with the new scenario: reverting to the string-based rank fails it with "(1,BG,v9) was not less than (1,BG,v2)" -- the mechanism itself -- and it passes with the fix. Full local suite 3576/0. --- .../code/api/util/ResourceDocRegistry.scala | 32 ++++++++++++++----- .../util/ResourceDocRegistryParityTest.scala | 22 +++++++++++++ 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala b/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala index 88104642fb..641c02ee48 100644 --- a/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala +++ b/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala @@ -73,7 +73,7 @@ object ResourceDocRegistry { ScannedApis.versionMapScannedApis.toSeq .collect { case (version: ScannedApiVersion, apis) if !explicit.contains(version) => version -> (() => apis.allResourceDocs.toSeq) } - .sortBy(entry => sortKey(entry._1)) + .sortBy(entry => sortKey(code.api.berlin.group.v1_3.OBP_BERLIN_GROUP_1_3_Alias.apiVersion)(entry._1)) .foldLeft(ListMap.empty[ApiVersion, () => Seq[ResourceDoc]])(_ + _) explicit ++ scanned } @@ -82,27 +82,43 @@ object ResourceDocRegistry { * Standards in ASCENDING precedence: a standard later in this list wins a partialFunctionName it * shares with an earlier one, because the `.toMap` consumers keep the last entry. This * reproduces the order of the hand-written union that preceded this registry (UK Open Banking, - * then Berlin Group). - * - * A standard that is not listed ranks below all of them -- including the Berlin Group v1.3 alias, - * whose apiStandard is whatever `berlin_group_v1_3_alias_path` names, so it can never override a - * first-class standard no matter how a deployment configures it. + * then Berlin Group). A standard that is not listed ranks below all of them. */ private val standardPrecedence: List[String] = List(ApiVersion.ukOpenBankingV20.apiStandard, ConstantsBG.berlinGroupVersion1.apiStandard) + /** Below every entry of standardPrecedence, whose lowest index is -1 for an unlisted standard. */ + private val derivedStandardRank: Int = -2 + /** * Total order over registry keys: precedence first, then the version's own identity. * + * `derivedAliasVersion` is the version of a standard that merely re-publishes another standard's + * docs -- today only the Berlin Group v1.3 alias. It is matched by identity, NOT by its + * apiStandard, because that string is the first segment of `berlin_group_v1_3_alias_path` and a + * deployment may legitimately choose one that an existing standard already uses: configured as + * "BG/v9" the alias would otherwise rank alongside Berlin Group and, sorting after "v2", let its + * re-stamped copies win getBalances, getAccountList and getAccountBalances away from the + * canonical docs it copied. Ranking it derivedStandardRank keeps that impossible for any + * configuration. + * * The tie-breaker is (apiStandard, apiShortVersion) rather than fullyQualifiedVersion because * that pair is exactly ScannedApiVersion's equals/hashCode key, so two distinct keys of a Map * keyed by version always differ in it and sortBy never has to fall back to the unordered input. * fullyQualifiedVersion concatenates the two (apiStandard.toUpperCase + apiShortVersion) and can * therefore collide across distinct keys -- ("BG", "v1.3") and ("BGV", "1.3") both render * "BGV1.3" -- which a deployment could reach through berlin_group_v1_3_alias_path. + * + * Curried and package-private so a test can rank against a synthetic alias without having to + * restart the JVM under a different berlin_group_v1_3_alias_path. */ - private def sortKey(version: ScannedApiVersion): (Int, String, String) = - (standardPrecedence.indexOf(version.apiStandard), version.apiStandard, version.apiShortVersion) + private[util] def sortKey(derivedAliasVersion: ScannedApiVersion) + (version: ScannedApiVersion): (Int, String, String) = { + val rank = + if (version == derivedAliasVersion) derivedStandardRank + else standardPrecedence.indexOf(version.apiStandard) + (rank, version.apiStandard, version.apiShortVersion) + } /** What the per-version resource-docs dispatcher serves for this version (empty if unknown). */ def docsFor(version: ApiVersion): Seq[ResourceDoc] = registry.get(version).map(_ ()).getOrElse(Nil) diff --git a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala index cbb5f9c624..3816545da8 100644 --- a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala +++ b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala @@ -1,5 +1,6 @@ package code.api.util +import code.api.berlin.group.ConstantsBG import code.api.berlin.group.v1_3.Http4sBGv13Alias import code.setup.ServerSetup import com.openbankproject.commons.util.{ApiStandards, ApiVersion, ScannedApiVersion} @@ -117,6 +118,27 @@ class ResourceDocRegistryParityTest extends ServerSetup { resolved.get("getAccountBalances") shouldBe Some("BGv2-getAccountBalances") } + // The Berlin Group v1.3 alias only re-publishes the canonical BG v1.3 docs, so it must never + // win a partialFunctionName away from the standard it copied. Its apiStandard is the first + // segment of berlin_group_v1_3_alias_path, so a deployment can point it at a name an existing + // standard already uses ("BG/v9"); ranking by that string alone put the alias alongside Berlin + // Group and, sorting after "v2", ahead of it. Ranking is by identity instead, and the synthetic + // alias below exercises the colliding configuration without needing a JVM under that prop. + scenario("a derived alias never outranks the standard it re-publishes", RegistryParityTag) { + val syntheticAlias = ScannedApiVersion("BG", "BG", "v9") + val rankOf = ResourceDocRegistry.sortKey(syntheticAlias) _ + withClue("the alias must sort before Berlin Group, i.e. lose the `.toMap` last-wins race ") { + rankOf(syntheticAlias) should be < rankOf(ConstantsBG.berlinGroupVersion2) + rankOf(syntheticAlias) should be < rankOf(ConstantsBG.berlinGroupVersion1) + } + withClue("the alias must also sort before UK Open Banking ") { + rankOf(syntheticAlias) should be < rankOf(ApiVersion.ukOpenBankingV401) + } + withClue("UK must still sort before Berlin Group, so BG keeps the names they share ") { + rankOf(ApiVersion.ukOpenBankingV401) should be < rankOf(ConstantsBG.berlinGroupVersion2) + } + } + // An unconfigured configuration-gated standard reports ScannedApiVersion("", "", ""), whose // fullyQualifiedVersion is "" as well. While ScannedApis kept it, ApiVersionUtils.valueOf("") // resolved successfully and GET /obp/v7.0.0/resource-docs//obp answered 200 with an empty From 759cc825d810c7db7e1e487cbd584110fd2e866c Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 10:36:02 +0200 Subject: [PATCH 54/59] fix: strict dependency validation unconditionally rejected every Java dynamic resource doc Two compounding bugs, both required to reproduce and both required to fix, found by running the feature against a real packaged jar and a real Postgres database instead of the in-process test harness: 1. Every Java method_body implements Supplier> per convention. javac always erases that generic Supplier.get() to a synthetic bridge method (Object get()) whose body just invokevirtual-calls the real, properly-typed get() -- an ordinary same-class call. The dependency scanner's same-class exemption only recognised a Scala-specific mangled-name convention, so this call was treated as a dependency on a forbidden method: the dynamically-compiled class lives under the OBP-owned code.* package, but its randomly-generated name can never appear in a static whitelist. 2. CompiledObjects.validateDependency() re-validated this.partialFunction a second time after DynamicUtil.createJavaHttp4sEndpoint had already validated the real compiled Java class internally. For Java, this.partialFunction is OBP's own Http4sEndpointIO wrapper, not user code, and its bytecode legitimately calls internal OBP helpers (javaValueToJValue, logger, CustomJsonFormats.formats, compactRender) that were never meant to be whitelisted. Together these meant dynamic_code_compile_validate_enable=true rejected every Java doc unconditionally, benign or not -- invisible to the existing test suite because no scenario combined a genuinely benign Java body with strict validation turned on. Fix getDynamicCodeDependentMethods to recurse through any same-class self-call rather than treat it as a leaf dependency, and make CompiledObjects.validateDependency() a no-op for the Java language (the real validation already happened inside createJavaHttp4sEndpoint). --- .../endpoint/helper/DynamicEndpoints.scala | 15 +++++- .../scala/code/api/util/DynamicUtil.scala | 15 +++++- ...esourceDocJavaSecurityValidationTest.scala | 50 +++++++++++++++++++ 3 files changed, 78 insertions(+), 2 deletions(-) diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala index 5d3ef7c1cc..97b5c34459 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala @@ -174,8 +174,21 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo * this will check all the dynamic scala code dependencies at compile time. * *Search for the usage, you can see how to use it in OBP code. + * + * Scala-only: for the Scala language, `this.partialFunction` IS the compiled user code, so + * validating its bytecode directly is correct. For Java, `this.partialFunction` is instead + * OBP's own Http4sEndpointIO wrapper (built by DynamicUtil.createJavaHttp4sEndpoint) around the + * real compiled Java class -- its bytecode legitimately calls internal OBP helpers + * (DynamicUtil.javaValueToJValue/logger, CustomJsonFormats.formats, JsonAliases.compactRender) + * that were never meant to be dependency-whitelisted, since they are framework glue, not + * user-supplied code. createJavaHttp4sEndpoint already validates the real compiled Java class + * internally (see its own doc comment) before ever returning that wrapper, so re-validating the + * wrapper here is both redundant and wrong -- it would reject every Java doc unconditionally. */ - def validateDependency() = Validation.validateDependency(this.partialFunction) + def validateDependency() = programmingLang match { + case "java" | "Java" => () + case _ => Validation.validateDependency(this.partialFunction) + } /** * This is used to check the security permission at the run time. diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index fee661611b..f572fe55bd 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -199,7 +199,20 @@ object DynamicUtil extends MdcLoggable{ ternary @ (typeName, methodName, signature) <- APIUtil.getDependentMethods(className, method.getName, method.getSignature, force) } yield { // if method is also dynamic compile code, extract it's dependent method - if(className.startsWith(typeName) && methodName.startsWith(clazz.getPackage.getName+ "$")) { + if (typeName == className) { + // A same-class self-call is not itself a dependency to police -- recurse into what the + // target method calls instead of flagging the call as forbidden. This is required for + // Java: every Java dynamic resource doc implements Supplier> + // (the documented convention), and the compiler always erases that generic Supplier.get() + // to a synthetic bridge method `Object get()` whose body is just `return this.get();` -- + // an ordinary same-class invokevirtual call to the real, properly-typed get(). Without this + // branch that bridge call is scanned like any other, resolves to (thisClass, "get", ...), + // and since the dynamically-compiled class lives under the OBP-owned code.* package + // (isRestrictedType) but its randomly-generated name can never appear in a static + // whitelist, EVERY Java doc is unconditionally rejected under strict validation -- + // regardless of what its code actually does. + listBuffer.appendAll(APIUtil.getDependentMethods(typeName, methodName, signature, force)) + } else if(className.startsWith(typeName) && methodName.startsWith(clazz.getPackage.getName+ "$")) { listBuffer.appendAll(APIUtil.getDependentMethods(typeName, methodName, signature, force)) } else { listBuffer.append(ternary) diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala index 66fabae06b..72f6cc6934 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala @@ -61,9 +61,59 @@ class DynamicResourceDocJavaSecurityValidationTest extends V400ServerSetup { ) } + // Every Java method_body implements Supplier> per convention (see + // DynamicUtil.createJavaHttp4sEndpoint's doc comment). javac always erases that generic + // Supplier.get() to a synthetic bridge method `Object get()` whose body just invokevirtual-calls + // the real, properly-typed get() -- an ordinary same-class call regardless of what the body does. + private def benignMethodBody: String = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaSecurityBenignProbe implements Supplier> { + | private Object apply(Object[] args) { + | Map response = new LinkedHashMap<>(); + | response.put("greeting", "hello"); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + private def createRequest = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) feature("Security validation of Java method_body against dynamic_code_compile_validate_dependencies") { + + // Regression guard: the Supplier.get() generics-erasure bridge method's same-class call to the + // real get() must not itself be treated as a call to a forbidden method. Without this, every + // Java doc -- malicious or not -- was rejected under strict validation, because the compiled + // class lives under the OBP-owned code.* package but its randomly-generated name can never + // appear in a static whitelist. + scenario("Registering a benign Java doc succeeds even with strict validation enabled") { + enableStrictValidation() + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "benignProbeTest", + requestUrl = "/benign_probe_test/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(benignMethodBody, "UTF-8"), + programmingLang = "Java" + ) + val resp = makePostRequest(createRequest, write(doc)) + + Then("the compile succeeds -- the Supplier.get() bridge method's self-call is not a forbidden dependency") + resp.code should equal(201) + } scenario("Registering a Java doc that calls a non-whitelisted OBP method is rejected with 400") { enableStrictValidation() Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) From adea5e5c25240f2d34644875c98ed6f6d48fd2f3 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 16:39:54 +0200 Subject: [PATCH 55/59] fix: recurse through multi-level same-class calls in Java dependency scan The Supplier.get() bridge-method exemption added for Java dynamic resource docs only unrolled one level: it appended a same-class method's own callees as raw dependency tuples instead of recursing into them. A Java method_body that factors logic into its own private helper methods (fully ordinary Java) reintroduces the exact same false rejection one level deeper -- the un-recursed helper's callees get flagged as calls to the compiled class's own randomly-generated, unwhitelistable name. Replace the one-level unroll with a proper recursive expansion that follows same-class calls to arbitrary depth until a genuinely foreign dependency is reached, with a visited-set cycle guard: direct or mutual recursion between same-class private methods (e.g. a factorial/fibonacci helper) is ordinary Java and must not recurse forever, and a cycle correctly contributes nothing further rather than falling back to a leaf that gets rejected the same way. Applies uniformly to both the new Java same-class case and the pre-existing Scala nested-closure case, which had the identical one-level limitation. --- .../scala/code/api/util/DynamicUtil.scala | 58 ++++++++++++------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 593f39a7a9..ef26937b6b 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -193,30 +193,48 @@ object DynamicUtil extends MdcLoggable{ val classPool = getClassPool(clazz.getClassLoader) //NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. val ctClass = classPool.get(className) + + // A same-class (or same-generated-unit, for the Scala nested-closure case below) call is not + // itself a dependency to police -- recurse into what the TARGET method calls instead of + // flagging the call itself as forbidden, all the way down until a genuinely foreign + // dependency is reached. This is required for Java: every Java dynamic resource doc + // implements Supplier> (the documented convention), and the + // compiler always erases that generic Supplier.get() to a synthetic bridge method + // `Object get()` whose body is just `return this.get();` -- an ordinary same-class + // invokevirtual call to the real, properly-typed get(). A single level of unrolling only + // fixes that one hop: any Java body that factors logic into its own private helper methods + // (an entirely normal thing to do) reintroduces the exact same false rejection one level + // deeper, since the un-recursed helper's own callees would otherwise be appended as raw + // (thisClass, method) tuples and then rejected as calls to an unwhitelistable random-UUID + // class. `visited` guards against a call cycle -- direct or mutual recursion between + // same-class private methods (e.g. a fibonacci/factorial helper) is entirely normal Java and + // would otherwise recurse forever. On hitting a cycle this contributes nothing further (Nil), + // not a leaf: the recursive call is still a same-class call, not a foreign dependency, and + // whatever it in turn depends on is already being expanded by the in-progress call further up + // this same path -- returning it as a leaf here would flag the method's own name + // (unwhitelistable, like any other randomly-named dynamic class) as a forbidden dependency, + // exactly the bug this whole function exists to avoid. + def expand(typeName: String, methodName: String, signature: String, visited: Set[(String, String, String)]): List[(String, String, String)] = { + val key = (typeName, methodName, signature) + val sameUnit = typeName == className || + (className.startsWith(typeName) && methodName.startsWith(clazz.getPackage.getName + "$")) + if (!sameUnit) { + List(key) + } else if (visited.contains(key)) { + Nil + } else { + APIUtil.getDependentMethods(typeName, methodName, signature, force).flatMap { case (t, m, s) => + expand(t, m, s, visited + key) + } + } + } + for { method <- ctClass.getDeclaredMethods.toList if predicate(method.getName) - ternary @ (typeName, methodName, signature) <- APIUtil.getDependentMethods(className, method.getName, method.getSignature, force) + (typeName, methodName, signature) <- APIUtil.getDependentMethods(className, method.getName, method.getSignature, force) } yield { - // if method is also dynamic compile code, extract it's dependent method - if (typeName == className) { - // A same-class self-call is not itself a dependency to police -- recurse into what the - // target method calls instead of flagging the call as forbidden. This is required for - // Java: every Java dynamic resource doc implements Supplier> - // (the documented convention), and the compiler always erases that generic Supplier.get() - // to a synthetic bridge method `Object get()` whose body is just `return this.get();` -- - // an ordinary same-class invokevirtual call to the real, properly-typed get(). Without this - // branch that bridge call is scanned like any other, resolves to (thisClass, "get", ...), - // and since the dynamically-compiled class lives under the OBP-owned code.* package - // (isRestrictedType) but its randomly-generated name can never appear in a static - // whitelist, EVERY Java doc is unconditionally rejected under strict validation -- - // regardless of what its code actually does. - listBuffer.appendAll(APIUtil.getDependentMethods(typeName, methodName, signature, force)) - } else if(className.startsWith(typeName) && methodName.startsWith(clazz.getPackage.getName+ "$")) { - listBuffer.appendAll(APIUtil.getDependentMethods(typeName, methodName, signature, force)) - } else { - listBuffer.append(ternary) - } + listBuffer.appendAll(expand(typeName, methodName, signature, Set.empty)) } listBuffer.distinct.toList From b745291ce316f6a1fc28f7f86456b36bf1d5b225 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 16:40:52 +0200 Subject: [PATCH 56/59] fix: reject unsupported programming_lang in the v6 validate endpoint POST /obp/v6.0.0/management/dynamic-resource-docs/validate re-implemented the request_verb and example_request_body preconditions the v4 create endpoint checks, but not the programming_lang check added alongside Java support. CompiledObjects falls through to the Scala compile path for any value it doesn't recognise as "java"/"Java", so validate would report valid=true for a body that happens to compile as Scala under a bogus or misspelled programming_lang -- a verdict that contradicts what the create endpoint would actually do (400 DynamicCodeLangNotSupport), defeating the point of a dry-run validation endpoint. Add the same language check, mirroring Http4s400's validateDynamicResourceDocBody. No prior test exercised this endpoint at all; added coverage for the rejection and for both supported languages. --- .../scala/code/api/v6_0_0/Http4s600.scala | 11 +++ .../ValidateDynamicResourceDocTest.scala | 81 +++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 obp-api/src/test/scala/code/api/v6_0_0/ValidateDynamicResourceDocTest.scala diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index bc5559d9b6..d3a2d4dc6d 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -4427,6 +4427,17 @@ object Http4s600 { case _ => true } } + // Mirrors Http4s400's validateDynamicResourceDocBody: fail fast on an unsupported + // programming_lang here too, rather than reporting `valid = true` for a language + // create would actually reject with 400 DynamicCodeLangNotSupport (CompiledObjects + // silently falls through to the Scala compile path for any value it doesn't + // recognise as Java, so an unsupported/misspelled language would otherwise still + // "validate" successfully as Scala). + _ <- Helper.booleanToFuture( + s"""$DynamicCodeLangNotSupport programming_lang ${body.programmingLang}, currently supported languages: Scala, Java""", + cc = Some(cc)) { + Set("", "scala", "Scala", "java", "Java").contains(body.programmingLang) + } } yield try { code.api.dynamic.endpoint.helper.CompiledObjects( body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang).validateDependency() diff --git a/obp-api/src/test/scala/code/api/v6_0_0/ValidateDynamicResourceDocTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/ValidateDynamicResourceDocTest.scala new file mode 100644 index 0000000000..7637619c6b --- /dev/null +++ b/obp-api/src/test/scala/code/api/v6_0_0/ValidateDynamicResourceDocTest.scala @@ -0,0 +1,81 @@ +package code.api.v6_0_0 + +import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.util.ApiRole +import code.entitlement.Entitlement +import org.json4s.native.Serialization.write + +/** + * `POST /obp/v6.0.0/management/dynamic-resource-docs/validate` must reject an unsupported + * `programming_lang` the same way `POST .../dynamic-resource-docs` (create) does, rather than + * reporting `valid = true` for a language create would actually 400 on -- see + * Http4s600.validateDynamicResourceDoc's own doc comment: CompiledObjects falls through to the + * Scala compile path for any programming_lang value it doesn't recognise as Java, so a body that + * happens to be valid Scala would otherwise "validate" successfully under a bogus/misspelled + * language. + */ +class ValidateDynamicResourceDocTest extends V600ServerSetup { + + private def validateRequest = (v6_0_0_Request / "management" / "dynamic-resource-docs" / "validate").POST <@ (user1) + + private def docWith(programmingLang: String) = + SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + programmingLang = programmingLang + ) + + feature("Validate Dynamic Resource Doc rejects an unsupported programming_lang") { + scenario("An unsupported programming_lang is rejected, not silently validated as Scala") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val resp = makePostRequest(validateRequest, write(docWith("Python"))) + + Then("the request is rejected with 400, not a 200 valid=true/false body") + resp.code should equal(400) + resp.body.toString should include("OBP-40049") + } + + scenario("programming_lang \"Scala\" is accepted (baseline, unaffected by the language check)") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val resp = makePostRequest(validateRequest, write(docWith("Scala"))) + + Then("the request reaches the compile step and responds 200") + resp.code should equal(200) + (resp.body \ "valid").values should equal(true) + } + + scenario("programming_lang \"Java\" is accepted (baseline, unaffected by the language check)") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val javaBody = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class ValidateEndpointJavaProbe implements Supplier> { + | private Object apply(Object[] args) { + | Map response = new LinkedHashMap<>(); + | response.put("greeting", "hello"); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + val doc = docWith("Java").copy(methodBody = java.net.URLEncoder.encode(javaBody, "UTF-8")) + val resp = makePostRequest(validateRequest, write(doc)) + + Then("the request reaches the compile step and responds 200") + resp.code should equal(200) + (resp.body \ "valid").values should equal(true) + } + } +} From 0fbd2f5279012d6a262735c80cbb0bf0e8cff515 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 16:42:23 +0200 Subject: [PATCH 57/59] fix: log a whitelist rejection distinctly from a genuine compile failure resourceDocs' per-row catch swallows any exception from toResourceDoc and logs it as "likely stored under the deprecated Lift contract -- re-author the body against the new native contract", regardless of cause. That is accurate for a genuine compile failure, but CompiledObjects.validateDependency now runs fresh on every construction (not just at create/update time), so a previously-registered doc can start throwing JsonResponseException here too -- purely because dynamic_code_compile_validate_dependencies was tightened after it was registered. The endpoint silently drops from the listing either way, but the log sends whoever reads it toward rewriting a body that was never the problem. Catch JsonResponseException separately and report the actual whitelist rejection reason (via the same JsonResponseExtractor pattern Http4s600's validate endpoint already uses for this exception shape), falling back to the existing generic message for every other cause. No dedicated test: this only changes log wording for a rejection path already covered end-to-end by DynamicResourceDocJavaSecurityValidationTest; asserting on log content here would need capture infrastructure this codebase doesn't otherwise use for a diagnostics-only change. --- .../DynamicResourceDocsEndpointGroup.scala | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala index 71b33776fd..a20ce39ea1 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala @@ -21,6 +21,22 @@ object DynamicResourceDocsEndpointGroup extends EndpointGroup with code.util.Hel try { Some(toResourceDoc(dynamicDoc)) } catch { + // Validation.validateDependency / createJavaHttp4sEndpoint's own rejection path both throw + // this specifically for a dependency-whitelist miss -- distinct from a genuine compile + // failure, and reachable here (not just at create/update time) because CompiledObjects' + // validation runs fresh on every construction and dynamic_code_compile_validate_dependencies + // can be tightened after a doc was already registered. Logging it as a "deprecated Lift + // contract" problem sends whoever reads this log to re-author a body that is not the + // problem, instead of at the whitelist they (or someone else) just edited. + case e: code.api.JsonResponseException => + val reason = e.jsonResponse match { + case APIUtil.JsonResponseExtractor(msg, _) => msg + case _ => Option(e.getMessage).getOrElse("") + } + logger.error(s"[DynamicResourceDocsEndpointGroup] skipping dynamic resource doc '${dynamicDoc.requestVerb} ${dynamicDoc.requestUrl}' " + + s"(id=${dynamicDoc.dynamicResourceDocId.getOrElse("")}, programming_lang=${dynamicDoc.programmingLang}): rejected by dependency " + + s"validation (dynamic_code_compile_validate_dependencies). $reason") + None case e: Throwable => logger.error(s"[DynamicResourceDocsEndpointGroup] skipping dynamic resource doc '${dynamicDoc.requestVerb} ${dynamicDoc.requestUrl}' " + s"(id=${dynamicDoc.dynamicResourceDocId.getOrElse("")}): its methodBody could not be compiled under the native http4s contract. " + From 7b6057f930171f2b8205103400d8d9c743f923ce Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 16:44:13 +0200 Subject: [PATCH 58/59] perf: bind allowedCompilationMethods once per validateDependency call allowedCompilationMethods was changed from val to def so it observes a live props override (needed for setPropsValues-driven tests), but validateDependency's collect guard reads it twice per dependency tuple -- Map.get and a separate .exists scan. Each read re-derives the whitelist from scratch: a prop read, a regex-based rewrite of the whitelist source into compilable Scala, and a ConcurrentHashMap lookup keyed on that ~1.5KB string. For N dependency tuples that is up to 2N re-derivations where one would do. Bind it to a val at the top of the method instead. Still observes props changes between calls (each validateDependency invocation re-binds), just not multiple times within the same call. --- obp-api/src/main/scala/code/api/util/DynamicUtil.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index ef26937b6b..694cf56145 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -515,6 +515,11 @@ object DynamicUtil extends MdcLoggable{ * Here only validate the restricted types(isObpClass + val restrictedTypes), not all scala/java types. */ private def validateDependency(dependentMethods: List[(String, String, String)]) = { + // Bound once per call, not re-derived per dependency tuple: allowedCompilationMethods is a + // def (see the "def, not val" comment above) so it observes a live props change, but it + // recompiles the whitelist source on every access -- reading it twice per element inside + // the `collect` guard below would mean up to 2N re-derivations for N dependency tuples. + val allowedCompilationMethods = this.allowedCompilationMethods val notAllowedDependentMethods = dependentMethods collect { case (typeName, method, _) if isRestrictedType(typeName) && From 6bb5223eee95ff18392299fd5878ac889e710970 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 16:45:28 +0200 Subject: [PATCH 59/59] fix: fail fast when a compiled Java class's bytecode is missing createJavaHttp4sEndpoint reflectively reads the compiled class's bytes out of MemoryClassLoader's private mapClassBytes map to hand them to Javassist for dependency validation. The lookup result was passed straight to ByteArrayClassPath unchecked: if a future java-scriptengine version keys that map differently (internal name vs binary name, or drops the field entirely), get() returns null, and the failure only surfaces later as an opaque NPE inside Javassist -- nothing at the actual point of breakage indicates the cause was this reflective coupling to an internal field. Check for null and throw immediately with a message that names the field and the class it was looked up for. The throw happens inside the same Box tryo the rest of the compile step already runs in, so it is reported the same way any other compile failure is -- no behavior change on the success path. Also documents (no functional change) the pre-existing, unbounded-by-design memoClassPool/dynamicCompileResult caching tradeoff that the Java compile path now shares: each distinct compiled method_body retains its ClassLoader and ClassPool for the life of the process. This is consistent with the existing Scala compile cache's design (dynamic resource doc creation is gated behind operator-only roles, not open to arbitrary callers) rather than a new gap introduced here. --- .../scala/code/api/util/DynamicUtil.scala | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 694cf56145..c38de119c4 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -43,6 +43,16 @@ object DynamicUtil extends MdcLoggable{ } val toolBox: ToolBox[universe.type] = runtimeMirror(getClass.getClassLoader).mkToolBox() + // Neither this nor memoJavaCompiledScript below ever evicts, so each distinct ClassLoader (and + // therefore each distinct compiled Java method_body -- java-scriptengine hands createJavaHttp4sEndpoint + // a fresh MemoryClassLoader per compile) is retained for the life of the process, along with its + // ClassPool. This is the same unbounded-but-trusted-operator-only tradeoff dynamicCompileResult + // below already makes for the Scala compile cache, predating the Java path: registering a dynamic + // resource doc is gated behind canCreateDynamicResourceDoc / canCreateBankLevelDynamicResourceDoc, + // not open to arbitrary callers, and a served endpoint's ClassLoader must stay reachable for as + // long as that endpoint keeps serving requests -- an eviction policy here would need to be + // reference-counted against currently-registered docs to avoid reclaiming a live one, which is a + // larger change than this cache's existing (pre-Java) design accounted for. private val memoClassPool = new Memo[ClassLoader, ClassPool] // Caches only the compiled artifact (deterministic given the source string), never the // validation outcome built on top of it -- see createJavaHttp4sEndpoint's doc comment. @@ -753,6 +763,16 @@ object DynamicUtil extends MdcLoggable{ classBytesField.setAccessible(true) val classBytes = classBytesField.get(compiledClass.getClassLoader) .asInstanceOf[java.util.Map[String, Array[Byte]]].get(compiledClass.getName) + // Fail loudly and specifically here rather than handing Javassist a null byte array -- + // that would only surface later, inside ByteArrayClassPath/ClassPool, as an opaque NPE + // with no indication that the cause was this reflective read (e.g. a java-scriptengine + // upgrade that changes mapClassBytes' keying from binary name to internal name, or that + // stops using that field name at all). + if (classBytes == null) { + throw new IllegalStateException( + s"createJavaHttp4sEndpoint: MemoryClassLoader.mapClassBytes has no entry for " + + s"${compiledClass.getName} -- java-scriptengine's internal layout may have changed") + } getClassPool(compiledClass.getClassLoader) .appendClassPath(new javassist.ByteArrayClassPath(compiledClass.getName, classBytes))