diff --git a/.github/workflows/build_container.yml b/.github/workflows/build_container.yml index f9f87b6762..c339eed773 100644 --- a/.github/workflows/build_container.yml +++ b/.github/workflows/build_container.yml @@ -27,6 +27,9 @@ jobs: # -------------------------------------------------------------------------- compile: runs-on: ubuntu-latest + # The test job has carried a timeout since it was written; compile and report never did, + # so a hung Maven resolve blocks the build until GitHub's own 6-hour ceiling. + timeout-minutes: 25 steps: - uses: actions/checkout@v4 @@ -289,6 +292,9 @@ jobs: echo ResetPasswordUrlEnabled=true >> obp-api/src/main/resources/props/test.default.props echo consents.allowed=true >> obp-api/src/main/resources/props/test.default.props echo hikari.maximumPoolSize=20 >> obp-api/src/main/resources/props/test.default.props + # Enables the Berlin Group v1.3 alias so ResourceDocRegistryParityTest and + # ApiCollectionEndpointTest can exercise a real alias operation id end to end. + echo berlin_group_v1_3_alias_path=0.6/v1 >> obp-api/src/main/resources/props/test.default.props echo write_metrics=false >> obp-api/src/main/resources/props/test.default.props # Log emails instead of opening a real SMTP socket: without this, # LocalMappedConnector.sendCustomerNotification's EMAIL branch calls @@ -460,6 +466,7 @@ jobs: needs: test runs-on: ubuntu-latest if: always() + timeout-minutes: 10 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/build_pull_request.yml b/.github/workflows/build_pull_request.yml index 39716fe115..be8d159360 100644 --- a/.github/workflows/build_pull_request.yml +++ b/.github/workflows/build_pull_request.yml @@ -25,6 +25,9 @@ jobs: # -------------------------------------------------------------------------- compile: runs-on: ubuntu-latest + # The test job has carried a timeout since it was written; compile and report never did, + # so a hung Maven resolve blocks the build until GitHub's own 6-hour ceiling. + timeout-minutes: 25 steps: - uses: actions/checkout@v4 @@ -283,6 +286,9 @@ jobs: echo ResetPasswordUrlEnabled=true >> obp-api/src/main/resources/props/test.default.props echo consents.allowed=true >> obp-api/src/main/resources/props/test.default.props echo hikari.maximumPoolSize=20 >> obp-api/src/main/resources/props/test.default.props + # Enables the Berlin Group v1.3 alias so ResourceDocRegistryParityTest and + # ApiCollectionEndpointTest can exercise a real alias operation id end to end. + echo berlin_group_v1_3_alias_path=0.6/v1 >> obp-api/src/main/resources/props/test.default.props echo write_metrics=false >> obp-api/src/main/resources/props/test.default.props # Log emails instead of opening a real SMTP socket: without this, # LocalMappedConnector.sendCustomerNotification's EMAIL branch calls @@ -302,6 +308,17 @@ jobs: echo allow_user_generated_scala_code=true >> obp-api/src/main/resources/props/test.default.props - name: Run tests — shard ${{ matrix.shard }} (${{ matrix.name }}) + env: + # This job has declared a redis service since it was written, but nothing ever failed + # when the service was absent: ConcurrentRateLimiterRaceTest and + # MethodRoutingCacheInvalidationTest each `assume` a reachable Redis and cancel + # otherwise, and a cancelled test reports as a pass. Dropping the services: block, or + # a container that never became healthy, would have taken the rate-limiter and + # cache-invalidation races out of the run without changing a single report. + # + # RedisTestTarget turns that cancellation into a failure wherever this is set. + # Developers leave it unset and keep the skip. + OBP_TEST_REDIS_REQUIRED: "true" run: | # wildcardSuites requires comma-separated package prefixes (-w per entry). # The YAML >- scalar collapses newlines to spaces, so we convert here. @@ -453,6 +470,7 @@ jobs: needs: test runs-on: ubuntu-latest if: always() + timeout-minutes: 10 steps: - uses: actions/checkout@v4 diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala index b14f064629..a35a0483a4 100644 --- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala +++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala @@ -3,8 +3,6 @@ package code.api.ResourceDocs1_4_0 import code.api.Constant.{GET_DYNAMIC_RESOURCE_DOCS_TTL, GET_STATIC_RESOURCE_DOCS_TTL, HostName, PARAM_LOCALE} import code.api.OBPRestHelper import code.api.cache.Caching -import code.api.dynamic.endpoint.OBPAPIDynamicEndpoint -import code.api.dynamic.entity.OBPAPIDynamicEntity import code.api.util.APIUtil._ import code.api.util.ApiRole.{canReadDynamicResourceDocsAtOneBank, canReadResourceDoc} import code.api.util.ApiTag._ @@ -22,7 +20,6 @@ import code.api.v4_0_0.{APIMethods400, OBPAPI4_0_0} import code.api.v5_0_0.OBPAPI5_0_0 import code.api.v5_1_0.OBPAPI5_1_0 import code.api.v6_0_0.OBPAPI6_0_0 -import code.api.berlin.group.ConstantsBG import code.apicollectionendpoint.MappedApiCollectionEndpointsProvider import code.util.Helper import code.util.Helper.{MdcLoggable, ObpS, SILENCE_IS_GOLDEN} @@ -331,59 +328,15 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth logger.debug(s"getResourceDocsList says requestedApiVersion is $requestedApiVersion") - val resourceDocs = requestedApiVersion match { - case ApiVersion.v7_0_0 => code.api.v7_0_0.Http4s700.allResourceDocs // Use aggregated docs for v7.0.0 - case ConstantsBG.`berlinGroupVersion1` => code.api.berlin.group.v1_3.Http4sBGv13.resourceDocs - case ConstantsBG.`berlinGroupVersion2` => code.api.berlin.group.v2.Http4sBGv2.resourceDocs - case ApiVersion.v6_0_0 => code.api.util.http4s.Http4sResourceDocAggregation.v600 - case ApiVersion.v5_1_0 => code.api.util.http4s.Http4sResourceDocAggregation.v510 - case ApiVersion.v5_0_0 => code.api.util.http4s.Http4sResourceDocAggregation.v500 - case ApiVersion.v4_0_0 => code.api.util.http4s.Http4sResourceDocAggregation.v400 - case ApiVersion.v3_1_0 => code.api.util.http4s.Http4sResourceDocAggregation.v310 - case ApiVersion.v3_0_0 => code.api.util.http4s.Http4sResourceDocAggregation.v300 - case ApiVersion.v2_2_0 => code.api.util.http4s.Http4sResourceDocAggregation.v220 - case ApiVersion.v2_1_0 => code.api.util.http4s.Http4sResourceDocAggregation.v210 - case ApiVersion.v2_0_0 => code.api.util.http4s.Http4sResourceDocAggregation.v200 - case ApiVersion.v1_4_0 => code.api.util.http4s.Http4sResourceDocAggregation.v140 - case ApiVersion.v1_3_0 => code.api.util.http4s.Http4sResourceDocAggregation.v130 - case ApiVersion.v1_2_1 => code.api.util.http4s.Http4sResourceDocAggregation.v121 - case ApiVersion.`dynamic-endpoint` => OBPAPIDynamicEndpoint.allResourceDocs - case ApiVersion.`dynamic-entity` => OBPAPIDynamicEntity.allResourceDocs - case version: ScannedApiVersion => ScannedApis.versionMapScannedApis.get(version).map(_.allResourceDocs).getOrElse(ArrayBuffer.empty[ResourceDoc]) - case _ => ArrayBuffer.empty[ResourceDoc] - } + // ResourceDocRegistry is the single source of truth for both this per-version dispatch and + // APIUtil.allStaticResourceDocs' global operation-id union -- see that object's doc comment. + val resourceDocs = ResourceDocRegistry.docsFor(requestedApiVersion) logger.debug(s"There are ${resourceDocs.length} resource docs available to $requestedApiVersion") - val activeResourceDocs = requestedApiVersion match { - case ApiVersion.v7_0_0 => resourceDocs - case ConstantsBG.`berlinGroupVersion1` => resourceDocs // fully on http4s — no Lift route filter - case ConstantsBG.`berlinGroupVersion2` => resourceDocs - case ApiVersion.v1_2_1 => resourceDocs - case ApiVersion.v6_0_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v5_1_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v5_0_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v4_0_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v3_1_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v3_0_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v2_2_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v2_1_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v2_0_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v1_4_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.v1_3_0 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.`dynamic-entity` => resourceDocs // runtime CRUD now on Http4sDynamicEntity; routes are Nil, skip Lift-route filter - case ApiVersion.`dynamic-endpoint` => resourceDocs // dispatch now on Http4sDynamicEndpoint (proxy + native Piece C); routes carry only the stub, skip Lift-route filter - case ApiVersion.ukOpenBankingV20 => resourceDocs // fully on http4s — no Lift route filter - case ApiVersion.ukOpenBankingV31 => resourceDocs // fully on http4s — no Lift route filter - case _ => resourceDocs - } - - logger.debug(s"There are ${activeResourceDocs.length} resource docs available to $requestedApiVersion") - - val activePlusLocalResourceDocs = ArrayBuffer[ResourceDoc]() - activePlusLocalResourceDocs ++= activeResourceDocs + activePlusLocalResourceDocs ++= resourceDocs requestedApiVersion match { // only `obp` standard show the `localResourceDocs` diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala index 2ca0eeef91..a3c8cac50a 100644 --- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala +++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala @@ -3202,6 +3202,7 @@ object SwaggerDefinitionsJSON { operation_id = "OBPv4.0.0-getBanks", api_instance_id = "obp_node_a", consent_reference_id = Some(ExampleValue.consentReferenceIdExample.value), + auth_type = Some("Consent"), certificate_trust = Some("forwarded"), certificate_trust_detail = Some("cn=nginx-prod-1,ou=edge,o=tesobe gmbh,c=de") ) diff --git a/obp-api/src/main/scala/code/api/UKOpenBanking/v2_0_0/Http4sUKOBv200.scala b/obp-api/src/main/scala/code/api/UKOpenBanking/v2_0_0/Http4sUKOBv200.scala index f7a58e2c74..b6a2c56045 100644 --- a/obp-api/src/main/scala/code/api/UKOpenBanking/v2_0_0/Http4sUKOBv200.scala +++ b/obp-api/src/main/scala/code/api/UKOpenBanking/v2_0_0/Http4sUKOBv200.scala @@ -4,6 +4,7 @@ import cats.data.{Kleisli, OptionT} import cats.effect._ import code.api.util.APIUtil.ResourceDoc import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.util.Helper.MdcLoggable import com.openbankproject.commons.util.ApiVersion import org.http4s._ @@ -33,5 +34,5 @@ object Http4sUKOBv200 extends MdcLoggable { Http4sUKOBv200AIS.routes(req) } - val wrappedRoutes: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allRoutes) + val wrappedRoutes: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes)) } diff --git a/obp-api/src/main/scala/code/api/UKOpenBanking/v3_1_0/Http4sUKOBv310.scala b/obp-api/src/main/scala/code/api/UKOpenBanking/v3_1_0/Http4sUKOBv310.scala index 56a3c403b1..49c41d3573 100644 --- a/obp-api/src/main/scala/code/api/UKOpenBanking/v3_1_0/Http4sUKOBv310.scala +++ b/obp-api/src/main/scala/code/api/UKOpenBanking/v3_1_0/Http4sUKOBv310.scala @@ -4,6 +4,7 @@ import cats.data.{Kleisli, OptionT} import cats.effect._ import code.api.util.APIUtil.ResourceDoc import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.util.Helper.MdcLoggable import com.openbankproject.commons.util.ApiVersion import org.http4s._ @@ -74,5 +75,5 @@ object Http4sUKOBv310 extends MdcLoggable { .orElse(Http4sUKOBv310InternationalStandingOrders.routes(req)) } - val wrappedRoutes: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allRoutes) + val wrappedRoutes: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes)) } diff --git a/obp-api/src/main/scala/code/api/UKOpenBanking/v4_0_1/Http4sUKOBv401.scala b/obp-api/src/main/scala/code/api/UKOpenBanking/v4_0_1/Http4sUKOBv401.scala index c40362d530..4f9985f712 100644 --- a/obp-api/src/main/scala/code/api/UKOpenBanking/v4_0_1/Http4sUKOBv401.scala +++ b/obp-api/src/main/scala/code/api/UKOpenBanking/v4_0_1/Http4sUKOBv401.scala @@ -5,6 +5,7 @@ import cats.effect._ import code.api.util.APIUtil import code.api.util.APIUtil.ResourceDoc import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.util.Helper.MdcLoggable import com.openbankproject.commons.util.ApiVersion import org.http4s._ @@ -62,5 +63,5 @@ object Http4sUKOBv401 extends MdcLoggable { routes(req).map(_.putHeaders(Header.Raw(fapiInteractionIdHeader, interactionId))) } - val wrappedRoutes: HttpRoutes[IO] = withFapiInteractionId(ResourceDocMiddleware.apply(resourceDocs)(allRoutes)) + val wrappedRoutes: HttpRoutes[IO] = withFapiInteractionId(ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes))) } diff --git a/obp-api/src/main/scala/code/api/berlin/group/v1_3/Http4sBGv13.scala b/obp-api/src/main/scala/code/api/berlin/group/v1_3/Http4sBGv13.scala index a2d83e2737..a50c74b563 100644 --- a/obp-api/src/main/scala/code/api/berlin/group/v1_3/Http4sBGv13.scala +++ b/obp-api/src/main/scala/code/api/berlin/group/v1_3/Http4sBGv13.scala @@ -5,6 +5,7 @@ import cats.effect._ import code.api.berlin.group.ConstantsBG import code.api.util.APIUtil.ResourceDoc import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.util.Helper.MdcLoggable import org.http4s._ @@ -35,5 +36,5 @@ object Http4sBGv13 extends MdcLoggable { .orElse(Http4sBGv13SigningBaskets.routes(req)) } - val wrappedRoutes: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allRoutes) + val wrappedRoutes: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes)) } diff --git a/obp-api/src/main/scala/code/api/berlin/group/v1_3/OBP_BERLIN_GROUP_1_3_Alias.scala b/obp-api/src/main/scala/code/api/berlin/group/v1_3/OBP_BERLIN_GROUP_1_3_Alias.scala index 2beba04807..8fc872c200 100644 --- a/obp-api/src/main/scala/code/api/berlin/group/v1_3/OBP_BERLIN_GROUP_1_3_Alias.scala +++ b/obp-api/src/main/scala/code/api/berlin/group/v1_3/OBP_BERLIN_GROUP_1_3_Alias.scala @@ -47,8 +47,22 @@ import scala.collection.mutable.ArrayBuffer */ object OBP_BERLIN_GROUP_1_3_Alias extends OBPRestHelper with MdcLoggable with ScannedApis { + /** + * The version this aggregator registers under. + * + * `berlinGroupV13AliasPath` is empty when `berlin_group_v1_3_alias_path` is unset, so `.head` / + * `.last` must be guarded: this object is instantiated by the ScannedApis classpath scan, which + * catches a throwing companion and merely logs a warning, so an unguarded NoSuchElementException + * would drop the alias silently. Inactive registrations keep the empty-string version they have + * always had, which no request can address and which deliberately does NOT equal + * ConstantsBG.berlinGroupVersion1 -- colliding with the canonical BG v1.3 key would let this + * (doc-less) object win ScannedApis' `.toMap` and blank out /resource-docs/BGv1.3/obp. + */ override val apiVersion: ScannedApiVersion = - ScannedApiVersion(berlinGroupV13AliasPath.head, berlinGroupV13AliasPath.head, berlinGroupV13AliasPath.last) + if (berlinGroupV13AliasPath.nonEmpty) + ScannedApiVersion(berlinGroupV13AliasPath.head, berlinGroupV13AliasPath.head, berlinGroupV13AliasPath.last) + else + ScannedApiVersion("", "", "") val versionStatus: String = ApiVersionStatus.DRAFT.toString diff --git a/obp-api/src/main/scala/code/api/berlin/group/v2/Http4sBGv2.scala b/obp-api/src/main/scala/code/api/berlin/group/v2/Http4sBGv2.scala index 9e2d1ce640..7da4cdae90 100644 --- a/obp-api/src/main/scala/code/api/berlin/group/v2/Http4sBGv2.scala +++ b/obp-api/src/main/scala/code/api/berlin/group/v2/Http4sBGv2.scala @@ -4,28 +4,37 @@ import cats.data.{Kleisli, OptionT} import cats.effect._ import code.api.berlin.group.ConstantsBG import code.api.util.APIUtil.ResourceDoc +import code.api.util.ScannedApis import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.util.Helper.MdcLoggable +import com.openbankproject.commons.util.ScannedApiVersion import org.http4s._ import scala.collection.mutable.ArrayBuffer -object Http4sBGv2 extends MdcLoggable { +object Http4sBGv2 extends MdcLoggable with ScannedApis { type HttpF[A] = OptionT[IO, A] val implementedInApiVersion = ConstantsBG.berlinGroupVersion2 + // ScannedApis discovery marker: makes BGv2 convention-driven like the other Berlin Group / + // UK Open Banking standards, so ResourceDocRegistry picks it up without a hand-maintained entry. + override val apiVersion: ScannedApiVersion = implementedInApiVersion + val resourceDocs: ArrayBuffer[ResourceDoc] = Http4sBGv2AIS.resourceDocs ++ Http4sBGv2PIS.resourceDocs ++ Http4sBGv2PIIS.resourceDocs + override val allResourceDocs: ArrayBuffer[ResourceDoc] = resourceDocs + val allRoutes: HttpRoutes[IO] = Kleisli[HttpF, Request[IO], Response[IO]] { req => Http4sBGv2AIS.routes(req) .orElse(Http4sBGv2PIS.routes(req)) .orElse(Http4sBGv2PIIS.routes(req)) } - val wrappedRoutes: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allRoutes) + val wrappedRoutes: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes)) } diff --git a/obp-api/src/main/scala/code/api/cache/Redis.scala b/obp-api/src/main/scala/code/api/cache/Redis.scala index 05208e3360..cf1bdb89b4 100644 --- a/obp-api/src/main/scala/code/api/cache/Redis.scala +++ b/obp-api/src/main/scala/code/api/cache/Redis.scala @@ -7,7 +7,7 @@ import code.util.Helper.MdcLoggable import com.openbankproject.commons.ExecutionContext.Implicits.global import redis.clients.jedis.{Jedis, JedisPool, JedisPoolConfig} import scalacache.memoization.{cacheKeyExclude, memoizeF, memoizeSync} -import scalacache.{Cache, Flags} +import scalacache.{Cache, CacheConfig, DefaultCacheKeyBuilder, Flags} import scalacache.redis.RedisCache import scalacache.serialization.{Codec, FailedToDecode} import redis.clients.jedis.{Jedis, JedisPool, JedisPoolConfig} @@ -321,6 +321,50 @@ object Redis extends MdcLoggable { // building one per call only put two allocations in front of every cache read on the request // path. RedisCache is a thin wrapper over the pool built above and opens nothing of its own, so // the pool, its authentication and its SSL configuration stay shared. + /** + * The serialization identity these cached bytes were produced under. + * + * Cache entries are Kryo-encoded, and what Kryo produces depends on the Scala library and the + * chill build that encoded it. Two OBP-API versions compiled against different ones therefore + * write mutually unreadable bytes into the same keys -- and "unreadable" is the optimistic + * case. Measured across the 2.12 -> 2.13 migration: an EMPTY `List`, written by chill 0.9.3, + * decodes under 0.9.5 into a `scala.collection.immutable.Queue`. That decode SUCCEEDS. It is + * only at the call site, whose signature says `List`, that it fails -- + * + * class scala.collection.immutable.Queue cannot be cast to + * class scala.collection.immutable.List + * + * -- so the caller gets a 500 rather than a cache miss, and gets it for the whole TTL, because + * a failed read does not evict the entry. Reproduced on `GET /management/dynamic-message-docs` + * and `GET /management/connector-methods`: 200 on 2.12, 500 on 2.13 reading 2.12's entry, and + * fine in either version on its own. That is a rolling upgrade, or any upgrade against a warm + * Redis. + * + * The migration note anticipated the risk and described the consequence as a cold cache. For + * values that fail to decode that is exactly right. This handles the ones that do not fail. + * + * Namespacing the key is the fix rather than casting defensively at each call site: there are + * eight `List`-returning memoized methods today, the same drift can hit any other type, and no + * amount of care at the call sites can make bytes already in Redis readable. Entries written by + * another version simply stop being addressable and age out on their own TTL. + * + * The Scala binary version is the axis that moved here and is the one derived automatically. + * `obp.cache.serialization.version` is for the case it does not cover -- a dependency upgrade + * that changes the encoding without changing the Scala version, which is what chill 0.9.3 to + * 0.9.5 would have been on its own. Bump it in that situation; the cost is one cold cache. + */ + private val serializationNamespace: String = { + val scalaBinary = scala.util.Properties.versionNumberString.split('.').take(2).mkString(".") + val manual = APIUtil.getPropsValue("obp.cache.serialization.version", "1") + s"obpser$manual-scala$scalaBinary" + } + + // Prefixing happens here, in the key builder, rather than at the call sites: scalacache derives + // the rest of the key from the enclosing method and its arguments, and every caller goes through + // it. `memoizeSync` and `memoizeF` both read this same implicit config. + implicit val cacheConfig: CacheConfig = + CacheConfig(cacheKeyBuilder = DefaultCacheKeyBuilder(keyPrefix = Some(serializationNamespace))) + private val sharedCache: Cache[Any] = RedisCache[Any](jedisPool) private def cacheFor[A]: Cache[A] = sharedCache.asInstanceOf[Cache[A]] diff --git a/obp-api/src/main/scala/code/api/constant/constant.scala b/obp-api/src/main/scala/code/api/constant/constant.scala index d4bfed683b..80cfdaa6ba 100644 --- a/obp-api/src/main/scala/code/api/constant/constant.scala +++ b/obp-api/src/main/scala/code/api/constant/constant.scala @@ -16,6 +16,21 @@ object Constant extends MdcLoggable { final val directLoginHeaderName = "DirectLogin" + // createdByProcess of entitlement rows the consent engine copies onto a consent user — + // the per-consent principal a Consent-JWT authenticates as (its ResourceUser row carries + // CreatedByConsentId). Only rows tagged with this value may target a consent user: + // addEntitlement redirects any other grant to the consent's granting human, so durable + // roles (e.g. bank-creator grants) can never strand on a principal that dies with its + // consent. Also the marker for cleaning these rows up when the consent is revoked. + final val consent_user = "consent_user" + + // createdByProcess of entitlement rows granted through group membership (the Groups + // feature). The value predates this constant: the Groups feature originally wrote it to + // its own `process` column, a duplicate of createdByProcess since retired — provenance + // now lives in createdByProcess like every other granting mechanism, and group rows are + // identified by their group_id. + final val group_membership = "GROUP_MEMBERSHIP" + object Pagination { final val offset = 0 final val limit = 50 @@ -305,7 +320,12 @@ object Constant extends MdcLoggable { final val CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL: Int = APIUtil.getPropsValue(s"createLocalisedResourceDocJson.cache.ttl.seconds", "3600").toInt final val GET_DYNAMIC_RESOURCE_DOCS_TTL: Int = APIUtil.getPropsValue(s"dynamicResourceDocsObp.cache.ttl.seconds", "3600").toInt final val GET_STATIC_RESOURCE_DOCS_TTL: Int = APIUtil.getPropsValue(s"staticResourceDocsObp.cache.ttl.seconds", "3600").toInt - final val SHOW_USED_CONNECTOR_METHODS: Boolean = APIUtil.getPropsAsBoolValue(s"show_used_connector_methods", false) + // def, not final val: DynamicUtil.Validation.validateDependency (dynamic-code dependency + // checking) needs this to react to a props change without a restart -- e.g. test-time + // setPropsValues overrides. A final val here would freeze at whatever value was true the + // moment this object was first touched (typically during server boot, well before any test + // scenario runs), and no later prop override could ever reach it. + def SHOW_USED_CONNECTOR_METHODS: Boolean = APIUtil.getPropsAsBoolValue(s"show_used_connector_methods", false) // Rate Limiting Cache Prefixes (with global namespace and versioning) // Both call_counter and rl_active are versioned for consistent cache invalidation diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala index beaad62688..97b5c34459 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala @@ -77,7 +77,7 @@ trait EndpointGroup { * @param successResponseBody successResponseBody from the post json body,it is JValue here. * @param methodBody it is url-encoded string for the api level code. */ -case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String) { +case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String, programmingLang: String = "Scala") { val decodedMethodBody = URLDecoder.decode(methodBody, "UTF-8") val requestBody: Product = exampleRequestBody match { //this case means, we accept the empty string "" from json post body, we need to map it to None. @@ -87,7 +87,24 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo } val successResponse: Product = toCaseObject(successResponseBody) - private val partialFunction: Http4sEndpointIO = { + private val partialFunction: Http4sEndpointIO = programmingLang match { + case "java" | "Java" => + DynamicUtil.createJavaHttp4sEndpoint(decodedMethodBody) match { + case Full(func) => func + case Failure(msg: String, exception: Box[Throwable], _) => + throw exception.getOrElse(new RuntimeException(msg)) + case _ => throw new RuntimeException("compiled code return nothing") + } + case _ /* "Scala" | "scala" | "" | null, default */ => + scalaPartialFunction + } + + // Unchanged Scala-template compile path, factored out so the `partialFunction` match above stays + // readable. Only evaluated for Scala-language docs (the default) — Java-language docs never + // touch this, so example/response-body JValues that don't fit the Scala case-class generator + // (irrelevant for Java, since it doesn't use RequestRootJsonClass/ResponseRootJsonClass) are a + // non-issue there. + private def scalaPartialFunction: Http4sEndpointIO = { //If the requestBody is PrimaryDataBody, return None. otherwise, return the exampleRequestBody:Option[JValue] // In side OBP resourceDoc, requestBody and successResponse must be Product type， @@ -157,8 +174,21 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo * this will check all the dynamic scala code dependencies at compile time. * *Search for the usage, you can see how to use it in OBP code. + * + * Scala-only: for the Scala language, `this.partialFunction` IS the compiled user code, so + * validating its bytecode directly is correct. For Java, `this.partialFunction` is instead + * OBP's own Http4sEndpointIO wrapper (built by DynamicUtil.createJavaHttp4sEndpoint) around the + * real compiled Java class -- its bytecode legitimately calls internal OBP helpers + * (DynamicUtil.javaValueToJValue/logger, CustomJsonFormats.formats, JsonAliases.compactRender) + * that were never meant to be dependency-whitelisted, since they are framework glue, not + * user-supplied code. createJavaHttp4sEndpoint already validates the real compiled Java class + * internally (see its own doc comment) before ever returning that wrapper, so re-validating the + * wrapper here is both redundant and wrong -- it would reject every Java doc unconditionally. */ - def validateDependency() = Validation.validateDependency(this.partialFunction) + def validateDependency() = programmingLang match { + case "java" | "Java" => () + case _ => Validation.validateDependency(this.partialFunction) + } /** * This is used to check the security permission at the run time. diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala index 8e6350494b..a20ce39ea1 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala @@ -21,6 +21,22 @@ object DynamicResourceDocsEndpointGroup extends EndpointGroup with code.util.Hel try { Some(toResourceDoc(dynamicDoc)) } catch { + // Validation.validateDependency / createJavaHttp4sEndpoint's own rejection path both throw + // this specifically for a dependency-whitelist miss -- distinct from a genuine compile + // failure, and reachable here (not just at create/update time) because CompiledObjects' + // validation runs fresh on every construction and dynamic_code_compile_validate_dependencies + // can be tightened after a doc was already registered. Logging it as a "deprecated Lift + // contract" problem sends whoever reads this log to re-author a body that is not the + // problem, instead of at the whitelist they (or someone else) just edited. + case e: code.api.JsonResponseException => + val reason = e.jsonResponse match { + case APIUtil.JsonResponseExtractor(msg, _) => msg + case _ => Option(e.getMessage).getOrElse("") + } + logger.error(s"[DynamicResourceDocsEndpointGroup] skipping dynamic resource doc '${dynamicDoc.requestVerb} ${dynamicDoc.requestUrl}' " + + s"(id=${dynamicDoc.dynamicResourceDocId.getOrElse("")}, programming_lang=${dynamicDoc.programmingLang}): rejected by dependency " + + s"validation (dynamic_code_compile_validate_dependencies). $reason") + None case e: Throwable => logger.error(s"[DynamicResourceDocsEndpointGroup] skipping dynamic resource doc '${dynamicDoc.requestVerb} ${dynamicDoc.requestUrl}' " + s"(id=${dynamicDoc.dynamicResourceDocId.getOrElse("")}): its methodBody could not be compiled under the native http4s contract. " + @@ -49,7 +65,7 @@ object DynamicResourceDocsEndpointGroup extends EndpointGroup with code.util.Hel * */ private val toResourceDoc: JsonDynamicResourceDoc => ResourceDoc = { dynamicDoc => - val compiledObjects = CompiledObjects(dynamicDoc.exampleRequestBody, dynamicDoc.successResponseBody, dynamicDoc.methodBody) + val compiledObjects = CompiledObjects(dynamicDoc.exampleRequestBody, dynamicDoc.successResponseBody, dynamicDoc.methodBody, dynamicDoc.programmingLang) ResourceDoc( // partialFunction is a no-op stub — the runtime dispatch uses the native handler in // dynamicHttp4sFunction (the compiled artifact is OBPEndpointIO, not the Lift OBPEndpoint). diff --git a/obp-api/src/main/scala/code/api/sandbox/example_data/2016-04-28/example_import.json b/obp-api/src/main/scala/code/api/sandbox/example_data/2016-04-28/example_import.json index f6ee011d8f..96ed1ffe99 100644 --- a/obp-api/src/main/scala/code/api/sandbox/example_data/2016-04-28/example_import.json +++ b/obp-api/src/main/scala/code/api/sandbox/example_data/2016-04-28/example_import.json @@ -64,7 +64,7 @@ "currency":"GBP", "amount":"8084.32" }, - "IBAN":"BA12 1234 5123 4513 7599 6969 977", + "IBAN":"BA463990000000000001", "owners":["robert.xuk.x@example.com"], "generate_public_view":false, "generate_accountants_view":true, @@ -79,7 +79,7 @@ "currency":"GBP", "amount":"8084.32" }, - "IBAN":"BA12 1234 5123 4513 7599 6969 977", + "IBAN":"BA924990000000000001", "owners":["robert.yuk.y@example.com"], "generate_public_view":false, "generate_accountants_view":true, diff --git a/obp-api/src/main/scala/code/api/sandbox/example_data/example_import.json b/obp-api/src/main/scala/code/api/sandbox/example_data/example_import.json index 0126ddd054..85c2f9dcd1 100644 --- a/obp-api/src/main/scala/code/api/sandbox/example_data/example_import.json +++ b/obp-api/src/main/scala/code/api/sandbox/example_data/example_import.json @@ -64,7 +64,7 @@ "currency":"GBP", "amount":"6599.63" }, - "IBAN":"BA12 1234 5123 4518 4490 1189 877", + "IBAN":"BA511990000000000001", "owners":["Susan.X.0.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -79,7 +79,7 @@ "currency":"GBP", "amount":"6379.63" }, - "IBAN":"BA12 1234 5123 4511 8754 4625 177", + "IBAN":"BA241990000000000002", "owners":["Robert.X.0.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -94,7 +94,7 @@ "currency":"GBP", "amount":"7588.25" }, - "IBAN":"BA12 1234 5123 4510 4337 1399 677", + "IBAN":"BA941990000000000003", "owners":["Anil.X.0.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -109,7 +109,7 @@ "currency":"GBP", "amount":"6662.05" }, - "IBAN":"BA12 1234 5123 4514 4440 2184 977", + "IBAN":"BA671990000000000004", "owners":["Robert.X.0.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -124,7 +124,7 @@ "currency":"GBP", "amount":"3748.57" }, - "IBAN":"BA12 1234 5123 4518 9534 3427 277", + "IBAN":"BA401990000000000005", "owners":["Ellie.X.0.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -139,7 +139,7 @@ "currency":"GBP", "amount":"15860.50" }, - "IBAN":"BA12 1234 5123 4512 1957 2301 577", + "IBAN":"BA131990000000000006", "owners":["Anil.X.0.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -154,7 +154,7 @@ "currency":"GBP", "amount":"7724.41" }, - "IBAN":"BA12 1234 5123 4512 6914 8586 977", + "IBAN":"BA831990000000000007", "owners":["Anil.X.0.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -169,7 +169,7 @@ "currency":"GBP", "amount":"6599.63" }, - "IBAN":"BA12 1234 5123 4518 4490 1189 877", + "IBAN":"BA972990000000000001", "owners":["Susan.Y.9.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -184,7 +184,7 @@ "currency":"GBP", "amount":"6379.63" }, - "IBAN":"BA12 1234 5123 4511 8754 4625 177", + "IBAN":"BA702990000000000002", "owners":["Robert.Y.9.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -199,7 +199,7 @@ "currency":"GBP", "amount":"7588.25" }, - "IBAN":"BA12 1234 5123 4510 4337 1399 677", + "IBAN":"BA432990000000000003", "owners":["Anil.Y.9.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -214,7 +214,7 @@ "currency":"GBP", "amount":"6662.05" }, - "IBAN":"BA12 1234 5123 4514 4440 2184 977", + "IBAN":"BA162990000000000004", "owners":["Robert.Y.9.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -229,7 +229,7 @@ "currency":"GBP", "amount":"3748.57" }, - "IBAN":"BA12 1234 5123 4518 9534 3427 277", + "IBAN":"BA862990000000000005", "owners":["Ellie.Y.9.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -244,7 +244,7 @@ "currency":"GBP", "amount":"15860.50" }, - "IBAN":"BA12 1234 5123 4512 1957 2301 577", + "IBAN":"BA592990000000000006", "owners":["Anil.Y.9.GH"], "generate_public_view":true, "generate_accountants_view":true, @@ -259,7 +259,7 @@ "currency":"GBP", "amount":"7724.41" }, - "IBAN":"BA12 1234 5123 4512 6914 8586 977", + "IBAN":"BA322990000000000007", "owners":["Anil.Y.9.GH"], "generate_public_view":true, "generate_accountants_view":true, diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index 9e45f49432..7a317068ac 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -32,9 +32,6 @@ import cats.effect.IO import code.abacrule.AbacRuleEngine import code.accountholders.AccountHolders import code.api.Constant._ -import code.api.UKOpenBanking.v2_0_0.OBP_UKOpenBanking_200 -import code.api.UKOpenBanking.v3_1_0.OBP_UKOpenBanking_310 -import code.api.UKOpenBanking.v4_0_1.OBP_UKOpenBanking_401 import code.api._ import code.api.berlin.group.ConstantsBG import code.api.berlin.group.v1_3.JSONFactory_BERLIN_GROUP_1_3.{ErrorMessageBG, ErrorMessagesBG} @@ -4399,8 +4396,10 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ * * than the return value may be (getUserAndSessionContextFuture, ***,***),(map,***,***), (getOrElse,***,***) ...... */ - def getDependentMethods(className: String, methodName:String, signature: String): List[(String, String, String)] = { - if (SHOW_USED_CONNECTOR_METHODS) { + // force bypasses the SHOW_USED_CONNECTOR_METHODS gate below -- see + // DynamicUtil.getDynamicCodeDependentMethods' doc comment for why security validation needs this. + def getDependentMethods(className: String, methodName:String, signature: String, force: Boolean = false): List[(String, String, String)] = { + if (SHOW_USED_CONNECTOR_METHODS || force) { val methods = ListBuffer[(String, String, String)]() //NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. //eg: className == code.api.UKOpenBanking.v3_1_0.APIMethods_AccountAccessApi$$anonfun$createAccountAccessConsents$lzycompute$1 @@ -4759,7 +4758,14 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ } ) - val berlinGroupV13AliasPath = APIUtil.getPropsValue("berlin_group_v1_3_alias_path","").split("/").toList.map(_.trim) + // Empty segments are dropped so that "unset" really means "no alias". Without the filter an + // absent prop yields List("") -- "".split("/") returns Array(""), not an empty array -- which is + // nonEmpty, so every `if (berlinGroupV13AliasPath.nonEmpty)` guard downstream took its ACTIVE + // branch on a default instance: Http4sBGv13Alias published 55 docs stamped with the degenerate + // version ScannedApiVersion("", "", ""), whose operation ids came out as `BG-`, and its + // route bridge matched on the prefix "/" (every path) only to fall through again. + val berlinGroupV13AliasPath = + APIUtil.getPropsValue("berlin_group_v1_3_alias_path","").split("/").toList.map(_.trim).filter(_.nonEmpty) val getAtmsIsPublic = APIUtil.getPropsAsBoolValue("apiOptions.getAtmsIsPublic", true) @@ -4874,29 +4880,13 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ val allowedAnswerTransactionRequestChallengeAttempts = APIUtil.getPropsAsIntValue("answer_transactionRequest_challenge_allowed_attempts").openOr(3) - // Base is the v7 aggregation — the newest OBP-standard surface, which already contains the - // v6.0.0-and-older aggregation plus the v7-only endpoints (deduped by URL/method). Basing on - // the v6 aggregation silently excluded v7-only operation ids from everything that resolves - // operation ids through this list (api-collection endpoint validation, top-apis lookups, ...). - // ResourceDocRegistryParityTest pins the invariant that every per-standard surface the - // resource-docs dispatcher can serve is contained here. - lazy val allStaticResourceDocs = (code.api.v7_0_0.Http4s700.allResourceDocs - ++ OBP_UKOpenBanking_200.allResourceDocs - ++ OBP_UKOpenBanking_310.allResourceDocs - ++ OBP_UKOpenBanking_401.allResourceDocs - // Commented out: Lift endpoints migrated off / removed (Polish, STET, AUOpenBanking, MxOF/CNBV9, BahrainOBF) - // ++ code.api.Polish.v2_1_1_1.OBP_PAPI_2_1_1_1.allResourceDocs - // ++ code.api.STET.v1_4.OBP_STET_1_4.allResourceDocs - // ++ code.api.AUOpenBanking.v1_0_0.ApiCollector.allResourceDocs - // ++ code.api.MxOF.CNBV9_1_0_0.allResourceDocs - // ++ code.api.MxOF.OBP_MXOF_1_0_0.allResourceDocs - // ++ code.api.BahrainOBF.v1_0_0.ApiCollector.allResourceDocs - ++ code.api.berlin.group.v1_3.OBP_BERLIN_GROUP_1_3.allResourceDocs - // BGv2 was missing here even though /resource-docs/BGv2 serves it, so a BGv2 operation id - // (e.g. BGv2-getAccountDetails) failed the getAllResourceDocs membership check that - // api-collection-endpoints (and anything else resolving operation ids) relies on. - ++ code.api.berlin.group.v2.Http4sBGv2.resourceDocs).toList - + // Delegates to ResourceDocRegistry, the single source of truth shared with the per-version + // resource-docs dispatcher (ResourceDocsAPIMethods.getResourceDocsList) -- see that object's + // doc comment for why the two used to drift and how deriving both from one registry fixes it. + // Kept under this name so existing call sites (Http4s400, Http4s600, JSONFactory6.0.0, ...) + // don't need to move. + lazy val allStaticResourceDocs: List[ResourceDoc] = ResourceDocRegistry.allStaticResourceDocs + def allDynamicResourceDocs= (DynamicEntityHelper.doc ++ DynamicEndpointHelper.doc ++ DynamicEndpoints.dynamicResourceDocs).toList def getAllResourceDocs = allStaticResourceDocs ++ allDynamicResourceDocs diff --git a/obp-api/src/main/scala/code/api/util/ApiSession.scala b/obp-api/src/main/scala/code/api/util/ApiSession.scala index cdd812050d..fd125ef95e 100644 --- a/obp-api/src/main/scala/code/api/util/ApiSession.scala +++ b/obp-api/src/main/scala/code/api/util/ApiSession.scala @@ -39,7 +39,7 @@ case class CallContext( // the creator is the granting human (they create their own consent in the Portal). // Not set by Berlin Group / UK flows, where the consent may be created by a TPP flow // with no human logged in — see `consenter` for those. - // Read via humanUser / effectiveHumanUserId, where it takes precedence over consenter. + // Read via humanUser / accountableUserId, where it takes precedence over consenter. onBehalfOfUser: Box[User] = Empty, // The human (PSU) who AUTHORISED the consent this request runs under — the owner of // record, from the consent table's userId (bound by updateConsentUser during the @@ -113,7 +113,7 @@ case class CallContext( * Anything that must name a human rather than a principal reads this instead: the CBS adapter, * which tells the core banking system who is asking, and the consent ownership checks. * Stored data (metric rows included) always carries the authenticated principal; the human is - * resolved at read time via the consent table (see effectiveHumanUserId). + * resolved at read time via the consent table (see accountableUserId). */ def humanUser: Box[User] = onBehalfOfUser.or(consenter).or(user) @@ -182,7 +182,7 @@ case class CallContext( // (CallContext.user), never a resolved human. Under a consent that principal is the // consent's own shadow user (a per-consent UUID with an empty name) — the on-behalf-of // human is not stored here but resolved at read time via the consent table - // (consentReferenceId below -> consent.userId), see CallContext.effectiveHumanUserId. + // (consentReferenceId below -> consent.userId), see CallContext.accountableUserId. userId = this.user.map(_.userId).toOption, userName = this.user.map(_.name).toOption, consumerId = this.consumer.map(_.consumerId.get).toOption, @@ -217,22 +217,31 @@ case class CallContext( def userId: String = user.map(_.userId).openOrThrowException(AuthenticatedUserIsRequired) /** - * The human User this request is really about. + * The ACCOUNTABLE identity this request is really about — the user_id that durable + * state (creator role grants, account holders, entitlement requests) and attribution + * (metrics families, "my" queries) bind to. "Accountable" deliberately hints at a + * legal person: today resolution always ends at the human who granted the consent, + * but the contract is accountability, not species — if durable, sponsored agent + * identities are ever admitted as principals in their own right, resolution may stop + * at such an agent without this name becoming a lie (unlike the previous name, + * effectiveHumanUserId). * - * The authenticated `user` may be the human themselves, or an agent user minted by a - * Consent the human granted (e.g. Opey / MCP acting under a consent). Resolution order: + * The authenticated `user` may be the accountable party themselves, or a consent user + * minted by a Consent they granted (e.g. Opey / MCP acting under a consent) — consent + * users are ephemeral and must never hold durable state (see addEntitlement's guard). + * Resolution order: * 1. `onBehalfOfUser` or `consenter`, when a middleware populated them (free); * 2. otherwise resolve via the delegation registry: the caller's ResourceUser row's * CreatedByConsentId names the Consent that minted it, and that Consent's userId * names the granting human; - * 3. otherwise the caller IS the human. + * 3. otherwise the caller IS the accountable party. * * IMPORTANT: this reads only the authenticated user and server-written columns * (ResourceUser.CreatedByConsentId, MappedConsent.mUserId). It deliberately takes no * parameters so nothing caller-asserted (body/header/query values) can ever influence * the resolution — identity-sensitive queries (e.g. /my/banks) depend on that. */ - def effectiveHumanUserId: String = { + def accountableUserId: String = { val delegatedHumanUserId = onBehalfOfUser.or(consenter).map(_.userId).filter(_.nonEmpty) delegatedHumanUserId.openOr { val authenticatedUserId = user.map(_.userId).openOr("") diff --git a/obp-api/src/main/scala/code/api/util/ApiVersionUtils.scala b/obp-api/src/main/scala/code/api/util/ApiVersionUtils.scala index cc2641c67f..d291130498 100644 --- a/obp-api/src/main/scala/code/api/util/ApiVersionUtils.scala +++ b/obp-api/src/main/scala/code/api/util/ApiVersionUtils.scala @@ -2,7 +2,6 @@ package code.api.util import com.openbankproject.commons.util.ApiVersion._ import com.openbankproject.commons.util.ScannedApiVersion -import code.api.berlin.group.ConstantsBG object ApiVersionUtils { @@ -23,7 +22,6 @@ object ApiVersionUtils { v7_0_0 :: `dynamic-endpoint` :: `dynamic-entity` :: - ConstantsBG.berlinGroupVersion2 :: scannedApis ).distinct @@ -48,7 +46,6 @@ object ApiVersionUtils { case v7_0_0.fullyQualifiedVersion | v7_0_0.apiShortVersion => v7_0_0 case `dynamic-endpoint`.fullyQualifiedVersion | `dynamic-endpoint`.apiShortVersion => `dynamic-endpoint` case `dynamic-entity`.fullyQualifiedVersion | `dynamic-entity`.apiShortVersion => `dynamic-entity` - case version if version == ConstantsBG.berlinGroupVersion2.fullyQualifiedVersion || version == ConstantsBG.berlinGroupVersion2.apiShortVersion => ConstantsBG.berlinGroupVersion2 case version if(scannedApis.map(_.fullyQualifiedVersion).contains(version)) =>scannedApis.filter(_.fullyQualifiedVersion==version).head case version if(scannedApis.map(_.apiShortVersion).contains(version)) diff --git a/obp-api/src/main/scala/code/api/util/ConsentUtil.scala b/obp-api/src/main/scala/code/api/util/ConsentUtil.scala index f55cd0cfb3..f17d281d67 100644 --- a/obp-api/src/main/scala/code/api/util/ConsentUtil.scala +++ b/obp-api/src/main/scala/code/api/util/ConsentUtil.scala @@ -380,7 +380,10 @@ object Consent extends MdcLoggable { existingEntitlements.exists(_.roleName == entitlement.role_name) match { // Check is a role already added to a user case false => val bankId = if (role.requiresBankId) entitlement.bank_id else "" - Entitlement.entitlement.vend.addEntitlement(bankId, user.userId, entitlement.role_name) match { + // Tagged consent_user: this is the ONE writer allowed to target a consent + // user — addEntitlement redirects untagged grants to the granting human. + Entitlement.entitlement.vend.addEntitlement(bankId, user.userId, entitlement.role_name, + createdByProcess = Constant.consent_user) match { case Full(_) => (entitlement, "AddedOrExisted") case _ => (entitlement, CannotAddEntitlement + entitlement) @@ -905,7 +908,7 @@ object Consent extends MdcLoggable { } yield { (principal, callContext.copy( // The PSU stays reachable for everything that needs a human: the CBS adapter, metric - // attribution, and CallContext.effectiveHumanUserId. + // attribution, and CallContext.accountableUserId. consenter = Full(psu), ukConsentId = Some(storedConsent.consentId), consentReferenceId = Some(storedConsent.consentReferenceId) diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 69ca8dea3c..c38de119c4 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -43,7 +43,20 @@ object DynamicUtil extends MdcLoggable{ } val toolBox: ToolBox[universe.type] = runtimeMirror(getClass.getClassLoader).mkToolBox() + // Neither this nor memoJavaCompiledScript below ever evicts, so each distinct ClassLoader (and + // therefore each distinct compiled Java method_body -- java-scriptengine hands createJavaHttp4sEndpoint + // a fresh MemoryClassLoader per compile) is retained for the life of the process, along with its + // ClassPool. This is the same unbounded-but-trusted-operator-only tradeoff dynamicCompileResult + // below already makes for the Scala compile cache, predating the Java path: registering a dynamic + // resource doc is gated behind canCreateDynamicResourceDoc / canCreateBankLevelDynamicResourceDoc, + // not open to arbitrary callers, and a served endpoint's ClassLoader must stay reachable for as + // long as that endpoint keeps serving requests -- an eviction policy here would need to be + // reference-counted against currently-registered docs to avoid reclaiming a live one, which is a + // larger change than this cache's existing (pre-Java) design accounted for. private val memoClassPool = new Memo[ClassLoader, ClassPool] + // Caches only the compiled artifact (deterministic given the source string), never the + // validation outcome built on top of it -- see createJavaHttp4sEndpoint's doc comment. + private val memoJavaCompiledScript = new Memo[String, Box[ch.obermuhlner.scriptengine.java.JavaCompiledScript]] private def getClassPool(classLoader: ClassLoader) = memoClassPool.memoize(classLoader){ val cp = ClassPool.getDefault @@ -167,29 +180,71 @@ object DynamicUtil extends MdcLoggable{ } /** - * NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. + * NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. * @param clazz * @param predicate + * @param force bypasses the SHOW_USED_CONNECTOR_METHODS gate below. SHOW_USED_CONNECTOR_METHODS + * exists to opt in to an unrelated, expensive introspection/reporting feature (which + * connector methods a static endpoint touches) — it was never meant to gate SECURITY + * validation, which reuses this same bytecode scan. Without `force`, a deployment + * that sets dynamic_code_compile_validate_enable=true (the documented, security- + * relevant prop) but leaves the unrelated show_used_connector_methods at its default + * false would silently get an always-empty dependency list here — every dynamic-code + * call looks "allowed" no matter what it does, because there is nothing to check + * against the whitelist. Validation.validateDependency passes force=true so it is + * controlled solely by dynamic_code_compile_validate_enable, matching what an + * operator following that prop's own documentation would expect. * @return */ - def getDynamicCodeDependentMethods(clazz: Class[_], predicate: String => Boolean = _ => true): List[(String, String, String)] = - if (SHOW_USED_CONNECTOR_METHODS) { + def getDynamicCodeDependentMethods(clazz: Class[_], predicate: String => Boolean = _ => true, force: Boolean = false): List[(String, String, String)] = + if (SHOW_USED_CONNECTOR_METHODS || force) { val className = clazz.getTypeName val listBuffer = new ListBuffer[(String, String, String)]() val classPool = getClassPool(clazz.getClassLoader) - //NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. + //NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. val ctClass = classPool.get(className) + + // A same-class (or same-generated-unit, for the Scala nested-closure case below) call is not + // itself a dependency to police -- recurse into what the TARGET method calls instead of + // flagging the call itself as forbidden, all the way down until a genuinely foreign + // dependency is reached. This is required for Java: every Java dynamic resource doc + // implements Supplier> (the documented convention), and the + // compiler always erases that generic Supplier.get() to a synthetic bridge method + // `Object get()` whose body is just `return this.get();` -- an ordinary same-class + // invokevirtual call to the real, properly-typed get(). A single level of unrolling only + // fixes that one hop: any Java body that factors logic into its own private helper methods + // (an entirely normal thing to do) reintroduces the exact same false rejection one level + // deeper, since the un-recursed helper's own callees would otherwise be appended as raw + // (thisClass, method) tuples and then rejected as calls to an unwhitelistable random-UUID + // class. `visited` guards against a call cycle -- direct or mutual recursion between + // same-class private methods (e.g. a fibonacci/factorial helper) is entirely normal Java and + // would otherwise recurse forever. On hitting a cycle this contributes nothing further (Nil), + // not a leaf: the recursive call is still a same-class call, not a foreign dependency, and + // whatever it in turn depends on is already being expanded by the in-progress call further up + // this same path -- returning it as a leaf here would flag the method's own name + // (unwhitelistable, like any other randomly-named dynamic class) as a forbidden dependency, + // exactly the bug this whole function exists to avoid. + def expand(typeName: String, methodName: String, signature: String, visited: Set[(String, String, String)]): List[(String, String, String)] = { + val key = (typeName, methodName, signature) + val sameUnit = typeName == className || + (className.startsWith(typeName) && methodName.startsWith(clazz.getPackage.getName + "$")) + if (!sameUnit) { + List(key) + } else if (visited.contains(key)) { + Nil + } else { + APIUtil.getDependentMethods(typeName, methodName, signature, force).flatMap { case (t, m, s) => + expand(t, m, s, visited + key) + } + } + } + for { method <- ctClass.getDeclaredMethods.toList if predicate(method.getName) - ternary @ (typeName, methodName, signature) <- APIUtil.getDependentMethods(className, method.getName, method.getSignature) + (typeName, methodName, signature) <- APIUtil.getDependentMethods(className, method.getName, method.getSignature, force) } yield { - // if method is also dynamic compile code, extract it's dependent method - if(className.startsWith(typeName) && methodName.startsWith(clazz.getPackage.getName+ "$")) { - listBuffer.appendAll(APIUtil.getDependentMethods(typeName, methodName, signature)) - } else { - listBuffer.append(ternary) - } + listBuffer.appendAll(expand(typeName, methodName, signature, Set.empty)) } listBuffer.distinct.toList @@ -360,10 +415,46 @@ object DynamicUtil extends MdcLoggable{ object Validation { - val dynamicCodeSandboxPermissions = APIUtil.getPropsValue("dynamic_code_sandbox_permissions", "[]").trim - val scalaCodePermissioins = "List[java.security.Permission]"+dynamicCodeSandboxPermissions.replaceFirst("\\[","(").dropRight(1)+")" - val permissions:Box[List[java.security.Permission]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodePermissioins) - + // def, not val, throughout this object: these must react to a props change (e.g. test-time + // setPropsValues) without a restart, not freeze at whatever the props held the moment + // Validation was first touched (typically by whichever dynamic-code test happens to run + // first in a shared test JVM). This costs nothing extra in production -- the only expensive + // step, DynamicUtil.compileScalaCodeUnchecked, is already memoized by the exact source + // string, so re-evaluating these on every call is a cache hit unless the underlying props + // value actually changed. + // + // This makes allowedRuntimePermissions itself always current, but NOT everything downstream + // of it: Sandbox.sandbox(bankId) below separately caches the whole Sandbox it builds, keyed + // only by bankId -- so a bankId whose sandbox was already built keeps that snapshot of + // allowedRuntimePermissions until the process restarts, same staleness this def change fixed + // for validateDependency. Left as-is here because it's moot in practice: SecurityManager + // enforcement is already a no-op on this JVM (JEP 486, JDK 24+; see Sandbox's own comment), + // so neither the stale nor the fresh permission list is actually enforced. + + /** + * Turn the `dynamic_code_compile_validate_dependencies` props value into the Scala source + * that, once compiled, yields the whitelist. + * + * A named function rather than an inline expression so a test can drive the real thing. + * DynamicUtilTest used to hold a character-for-character copy of it, which meant the two + * could diverge with the test still green -- the copy was only kept in step here because + * whoever edited one happened to see the other. This is the only compile that happens + * reflectively at boot, so nothing at compile time would have caught the divergence either. + * + * `Map[String, String](` rather than `Map(`: the props default is an empty list, and a bare + * `Map()` leaves its type parameters undetermined, so the trailing `.toMap` cannot prove the + * elements are pairs and the reflective compilation fails. The `.toMap` is itself needed + * because `mapValues` returns a view rather than a Map on 2.13. + */ + def dependenciesScalaCode(dependenciesString: String): String = + s"${DynamicUtil.importStatements}" + + dependenciesString.replaceFirst("\\[", "Map[String, String](").dropRight(1) + + ").mapValues(v => StringUtils.split(v, ',').map(_.trim).toSet).toMap" + + def dynamicCodeSandboxPermissions = APIUtil.getPropsValue("dynamic_code_sandbox_permissions", "[]").trim + def scalaCodePermissioins = "List[java.security.Permission]"+dynamicCodeSandboxPermissions.replaceFirst("\\[","(").dropRight(1)+")" + def permissions:Box[List[java.security.Permission]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodePermissioins) + // all Permissions put at here // Here is the Java Permission document, please extend these permissions carefully. // https://docs.oracle.com/javase/8/docs/technotes/guides/security/spec/security-spec.doc3.html#17001 @@ -382,15 +473,11 @@ object DynamicUtil extends MdcLoggable{ // new RuntimePermission("accessDeclaredMembers"), // new RuntimePermission("getClassLoader"), // ) - val allowedRuntimePermissions = permissions.openOrThrowException("Can not compile the props `dynamic_code_sandbox_permissions` to permissions") - - val dependenciesString = APIUtil.getPropsValue("dynamic_code_compile_validate_dependencies", "[]").trim - // `Map[String, String](` rather than `Map(`: the props default is an empty list, and a bare - // `Map()` leaves its type parameters undetermined, so the trailing .toMap cannot prove the - // elements are pairs and the reflective compilation fails. The .toMap itself is needed because - // mapValues returns a view rather than a Map. - val scalaCodeDependencies = s"${DynamicUtil.importStatements}"+dependenciesString.replaceFirst("\\[","Map[String, String](").dropRight(1) +").mapValues(v => StringUtils.split(v, ',').map(_.trim).toSet).toMap" - val dependenciesBox: Box[Map[String, Set[String]]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodeDependencies) + def allowedRuntimePermissions = permissions.openOrThrowException("Can not compile the props `dynamic_code_sandbox_permissions` to permissions") + + def dependenciesString = APIUtil.getPropsValue("dynamic_code_compile_validate_dependencies", "[]").trim + def scalaCodeDependencies = dependenciesScalaCode(dependenciesString) + def dependenciesBox: Box[Map[String, Set[String]]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodeDependencies) /** * Compilation OBP Dependencies Guard, only checked the OBP methods, not scala/Java libraies(are checked during the runtime.). @@ -421,7 +508,7 @@ object DynamicUtil extends MdcLoggable{ // PractiseEndpoint.getClass.getTypeName + "*" -> "*", // // ).mapValues(v => StringUtils.split(v, ',').map(_.trim).toSet) - val allowedCompilationMethods: Map[String, Set[String]] = dependenciesBox.openOrThrowException("Can not compile the props `dynamic_code_compile_validate_dependencies` to Map") + def allowedCompilationMethods: Map[String, Set[String]] = dependenciesBox.openOrThrowException("Can not compile the props `dynamic_code_compile_validate_dependencies` to Map") //Do not touch this Set, try to use the `allowedPermissions` and `allowedMethods` to control the sandbox val restrictedTypes = Set( @@ -438,6 +525,11 @@ object DynamicUtil extends MdcLoggable{ * Here only validate the restricted types(isObpClass + val restrictedTypes), not all scala/java types. */ private def validateDependency(dependentMethods: List[(String, String, String)]) = { + // Bound once per call, not re-derived per dependency tuple: allowedCompilationMethods is a + // def (see the "def, not val" comment above) so it observes a live props change, but it + // recompiles the whitelist source on every access -- reading it twice per element inside + // the `collect` guard below would mean up to 2N re-derivations for N dependency tuples. + val allowedCompilationMethods = this.allowedCompilationMethods val notAllowedDependentMethods = dependentMethods collect { case (typeName, method, _) if isRestrictedType(typeName) && @@ -460,7 +552,9 @@ object DynamicUtil extends MdcLoggable{ def validateDependency(obj: AnyRef): Unit = { if(APIUtil.getPropsAsBoolValue("dynamic_code_compile_validate_enable",false)){ - val dependentMethods: List[(String, String, String)] = DynamicUtil.getDynamicCodeDependentMethods(obj.getClass) + // force=true: this check must not also require the unrelated show_used_connector_methods + // prop -- see getDynamicCodeDependentMethods' doc comment for why. + val dependentMethods: List[(String, String, String)] = DynamicUtil.getDynamicCodeDependentMethods(obj.getClass, force = true) validateDependency(dependentMethods) } else{ // If false, nothing to do here. ; @@ -543,4 +637,196 @@ object DynamicUtil extends MdcLoggable{ } } } + + /** + * Converts a plain value returned by a compiled Java `method_body` into a JValue, for endpoints + * where json4s' `Extraction.decompose` cannot help: it works by Scala-case-class/collection + * reflection, so a `java.util.Map`/`java.util.List` returned from Java decomposes to `{}`/`[]` + * (its entries are invisible to Scala reflection) rather than throwing — a silent data-loss bug, + * not a compile or runtime error, so it only surfaces as an empty response body. Recurses through + * the Java collection types directly; anything else (including a Scala case class constructed + * from Java, as ConnectorMethod's Java example does) falls back to Extraction.decompose. + */ + private def javaValueToJValue(value: Any): JValue = { + import scala.jdk.CollectionConverters._ + value match { + case null => JNull + case jv: JValue => jv + case m: java.util.Map[_, _] => + JObject(m.asScala.toList.map { case (k, v) => (String.valueOf(k), javaValueToJValue(v)) }) + case l: java.util.List[_] => + JArray(l.asScala.toList.map(javaValueToJValue)) + case s: String => JString(s) + case b: java.lang.Boolean => JBool(b) + case i: java.lang.Integer => JInt(BigInt(i.intValue())) + case l: java.lang.Long => JInt(BigInt(l.longValue())) + case d: java.lang.Double => JDouble(d.doubleValue()) + case f: java.lang.Float => JDouble(f.doubleValue()) + case bd: java.math.BigDecimal => JDecimal(BigDecimal(bd)) + case other => Extraction.decompose(other)(CustomJsonFormats.formats) + } + } + + /** + * Compiles a Java `method_body` for a DynamicResourceDoc endpoint into a native + * `Http4sEndpointIO` (`PartialFunction[Request[IO], CallContext => IO[Response[IO]]]`), the same + * type the Scala template compiles to in DynamicEndpoints.CompiledObjects. + * + * Reuses the same JSR-223 "java" engine (backed by a real javax.tools.JavaCompiler via + * ch.obermuhlner:java-scriptengine — see createJavaFunction above) and the same + * package-uniquification trick, but — unlike createJavaFunction, whose DynamicFunction shape is + * specific to the ConnectorMethod feature — wraps the compiled function in a hand-written + * Http4sEndpointIO here in Scala. The Java method_body never has to construct cats.effect.IO, + * org.http4s.Response, or a Scala PartialFunction: it only ever returns a plain Java object + * (Map/List/String/number/boolean/etc.), which this adapter serializes via javaValueToJValue + * above (NOT Extraction.decompose directly — see that method's doc comment for why). + * + * Java-side convention (identical to the existing ConnectorMethod convention): the pasted class + * implements java.util.function.Supplier>. The + * compiled function is invoked with: + * args(0) = the raw request body (String, or null if the request had none) + * args(1) = path params (java.util.Map) + * args(2) = the CallContext (present whenever this endpoint is actually being served) + * mirroring createJavaFunction's own `func(args ++ cc)` call (line above): appending an + * Option[CallContext] via `++` appends its *contents* (0 or 1 raw CallContext), not the Option + * wrapper itself, so Java reads args[2] directly as a CallContext, no unwrapping needed. + * + * Unlike createJavaFunction, this validates the actual compiled Java class (not just its Scala + * wrapper) against `dynamic_code_compile_validate_dependencies`/`dynamic_code_compile_validate_enable`. + * CompiledObjects.validateDependency() (called by the ResourceDoc-creation flow) only ever sees + * `this.partialFunction` — the hand-written Http4sEndpointIO below — whose own bytecode just + * calls `java.util.function.Function.apply`, a non-restricted type; it can't see what the pasted + * Java class does inside apply(Object[]). Worse, `func` itself (the Function returned by the + * pasted class's get()) is commonly a method reference (`this::apply`), which the JVM + * materialises as a synthetic lambda class whose bytecode is just a delegating call — validating + * `func.getClass` would be equally blind. So we go through the JSR-223 Compilable API directly + * (JavaScriptEngine implements it) instead of plain eval(), to get the real top-level compiled + * class/instance (JavaCompiledScript.getCompiledClass/getCompiledInstance) and validate that + * before the function is ever returned or invoked. + */ + def createJavaHttp4sEndpoint(methodBody: String): Box[code.api.util.APIUtil.Http4sEndpointIO] = + if (!dynamicCodeExecutionEnabled) Failure(ErrorMessages.DynamicCodeExecutionDisabled) + else { + import cats.effect.IO + import code.api.util.APIUtil.Http4sEndpointIO + import com.openbankproject.commons.ExecutionContext.Implicits.global + import com.openbankproject.commons.util.JsonAliases.compactRender + import org.http4s.headers.`Content-Type` + import org.http4s.dsl.io._ + import org.http4s.{MediaType, Request, Response} + + import scala.jdk.CollectionConverters._ + + // Only the compile step is memoized — deterministic given the same source string, and the + // one genuinely expensive part (a real javax.tools.JavaCompiler invocation). Dependency + // validation below is NOT memoized: it depends on mutable external config + // (dynamic_code_compile_validate_enable/_dependencies), which can change between two + // createJavaHttp4sEndpoint calls for the identical source string — e.g. a doc compiled once + // while validation was off, then a later create/update call resubmitting the exact same + // method_body after validation was turned on and the whitelist tightened. An earlier version + // of this function memoized the validated *result* (Box[Http4sEndpointIO]) as a single unit, + // so that second call silently reused the first call's unvalidated success — bypassing the + // now-stricter policy for any resubmitted source. Re-running validation on every call costs + // little: it is Javassist bytecode inspection plus a Map lookup, not another compile. + val compiledScriptBox: Box[ch.obermuhlner.scriptengine.java.JavaCompiledScript] = + memoJavaCompiledScript.memoize("java-http4s-endpoint:" + methodBody) { + // Real compile happens here (javax.tools.JavaCompiler via the JSR-223 "java" engine) — + // any Java syntax/type error surfaces as an exception, caught by this `Box tryo` and + // turned into a Failure. + Box tryo { + val packageExp = UUID.randomUUID().toString.replaceAll("^|-", "_") + val packageMatcher = Pattern.compile("""(?m)^\s*package\s+\S+?\s*;""").matcher(methodBody) + + val javaCode = s"""package code.api.util.dynamic.${packageExp}; + |${packageMatcher.replaceFirst("")} + |""".stripMargin + + val compiledScript = javaEngine.asInstanceOf[javax.script.Compilable].compile(javaCode) + .asInstanceOf[ch.obermuhlner.scriptengine.java.JavaCompiledScript] + + // getDynamicCodeDependentMethods loads a class's bytecode via Javassist's + // LoaderClassPath, which reads it through classLoader.getResourceAsStream(...). The + // compiler's ch.obermuhlner.scriptengine.java.MemoryClassLoader only overrides + // loadClass() — it never exposes the compiled bytes as a classpath resource — so that + // lookup silently fails (javassist.NotFoundException) and validation would see zero + // dependent methods no matter what the Java code actually calls. Read the bytes + // directly from the classloader's private byte map (reflection is unavoidable here: + // java-scriptengine exposes no public accessor) and hand them to Javassist explicitly + // via ByteArrayClassPath, so the real method bodies — including any restricted OBP + // call — are visible to validation. Done here, inside the compile memoization, so it + // runs exactly once per distinct source: ClassPool.appendClassPath has no dedup of its + // own, so doing this on every createJavaHttp4sEndpoint call (as an earlier version of + // this function did, on every resourceDocs-list rebuild for the process's lifetime) + // grew that ClassPool's classpath chain without bound. + val compiledClass = compiledScript.getCompiledClass + val classBytesField = compiledClass.getClassLoader.getClass.getDeclaredField("mapClassBytes") + classBytesField.setAccessible(true) + val classBytes = classBytesField.get(compiledClass.getClassLoader) + .asInstanceOf[java.util.Map[String, Array[Byte]]].get(compiledClass.getName) + // Fail loudly and specifically here rather than handing Javassist a null byte array -- + // that would only surface later, inside ByteArrayClassPath/ClassPool, as an opaque NPE + // with no indication that the cause was this reflective read (e.g. a java-scriptengine + // upgrade that changes mapClassBytes' keying from binary name to internal name, or that + // stops using that field name at all). + if (classBytes == null) { + throw new IllegalStateException( + s"createJavaHttp4sEndpoint: MemoryClassLoader.mapClassBytes has no entry for " + + s"${compiledClass.getName} -- java-scriptengine's internal layout may have changed") + } + getClassPool(compiledClass.getClassLoader) + .appendClassPath(new javassist.ByteArrayClassPath(compiledClass.getName, classBytes)) + + compiledScript + } + } + + // Deliberately outside compiledScriptBox's `Box tryo` AND outside the memoization above: a + // rejection here throws JsonResponseException, which must propagate UNCAUGHT (mirroring the + // Scala path's CompiledObjects.validateDependency(), also never wrapped in tryo) so + // compileDynamicResourceDoc's `case e: JsonResponseException => throw e` sees it intact. + // JsonResponseException never sets a Throwable message (getMessage == null); Box.tryo would + // catch it into Failure(null, Full(theException), Empty), and DynamicEndpoints.scala's + // `case Failure(msg: String, ...)` pattern silently fails to match a null msg — falling + // through to "compiled code return nothing" and discarding the real rejection reason. `.map` + // does not swallow exceptions the way `Box tryo` does, so this stays uncaught here. + compiledScriptBox.map { compiledScript => + // Validate the real compiled Supplier class before it's ever invoked — see the doc comment + // above for why this must run against getCompiledInstance, not `func`/`this.partialFunction`, + // and why it must run fresh on every call rather than being cached with the compile result. + Validation.validateDependency(compiledScript.getCompiledInstance) + + val func = compiledScript.eval().asInstanceOf[java.util.function.Function[Array[AnyRef], Any]] + val jsonContentType = `Content-Type`(MediaType.application.json) + + new Http4sEndpointIO { + override def isDefinedAt(req: Request[IO]): Boolean = true + + override def apply(req: Request[IO]): CallContext => IO[Response[IO]] = { cc => + val pathParams: java.util.Map[String, String] = cc.resourceDocument + .map(_.getPathParams(req.uri.path.segments.toList.map(_.encoded))) + .getOrElse(Map.empty[String, String]) + .asJava + + val valueIO: IO[Any] = IO.fromFuture(IO { + Future { + val args: Array[AnyRef] = Array(cc.httpBody.orNull, pathParams) + func(args ++ Some(cc)) + } + }) + + valueIO.flatMap { value => + Ok(compactRender(javaValueToJValue(value)), jsonContentType) + }.handleErrorWith { e => + logger.warn(s"createJavaHttp4sEndpoint: Java method_body threw", e) + InternalServerError( + compactRender(Extraction.decompose( + Map("code" -> 500, "message" -> s"OBP-50000: Unknown Error. ${e.getMessage}") + )(CustomJsonFormats.formats)), + jsonContentType + ) + } + } + } + } + } } diff --git a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala index 84b8fc9de8..627dd128d2 100644 --- a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala +++ b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala @@ -836,6 +836,7 @@ object ErrorMessages { val ChatMessageTypeNotAllowed = "OBP-39018: Invalid message_type. Allowed values: text, system." val SignalMessageTooLong = "OBP-39019: Signal message exceeds the maximum allowed length." val SignalMessageContainsDangerousCharacters = "OBP-39020: Signal message contains control or bidirectional-override characters, which are not allowed." + val SignalChannelNotFound = "OBP-39021: Signal Channel not found." // Transaction Request related messages (OBP-40XXX) val InvalidTransactionRequestType = "OBP-40001: Invalid value for TRANSACTION_REQUEST_TYPE" diff --git a/obp-api/src/main/scala/code/api/util/NewStyle.scala b/obp-api/src/main/scala/code/api/util/NewStyle.scala index 8145f828b9..ba7209efd6 100644 --- a/obp-api/src/main/scala/code/api/util/NewStyle.scala +++ b/obp-api/src/main/scala/code/api/util/NewStyle.scala @@ -3341,7 +3341,9 @@ object NewStyle extends MdcLoggable{ def createOrUpdateEndpointMapping(bankId: Option[String], endpointMapping: EndpointMappingT, callContext: Option[CallContext]) = { validateBankId(bankId, callContext) Future { - (EndpointMappingProvider.endpointMappingProvider.vend.createOrUpdate(bankId, endpointMapping), callContext) + val result = EndpointMappingProvider.endpointMappingProvider.vend.createOrUpdate(bankId, endpointMapping) + invalidateEndpointMappingCache() + (result, callContext) } map { i => (connectorEmptyResponse(i._1, callContext), i._2) } @@ -3350,12 +3352,48 @@ object NewStyle extends MdcLoggable{ def deleteEndpointMapping(bankId: Option[String], endpointMappingId: String, callContext: Option[CallContext]) = { validateBankId(bankId, callContext) Future { - (EndpointMappingProvider.endpointMappingProvider.vend.delete(bankId, endpointMappingId), callContext) + val result = EndpointMappingProvider.endpointMappingProvider.vend.delete(bankId, endpointMappingId) + invalidateEndpointMappingCache() + (result, callContext) } map { i => (connectorEmptyResponse(i._1, callContext), i._2) } } + /** + * Drop every memoized `getEndpointMappings(...)` entry after a mapping is created, + * updated, or deleted, so the change takes effect on the next request instead of waiting + * out `endpointMapping.cache.ttl.seconds`. Mirrors invalidateMethodRoutingCache: the + * memoize key embeds the literal method name, so one pattern delete clears every bankId + * variant. The pattern is a prefix of the actual name the macro renders + * (`getEndpointMappingsCached`), which is what makes one wildcard cover it. No-op / logged + * when Redis is unavailable (deleteKeysByPattern swallows and returns 0). + * + * This became necessary with the cache key fix below. While callContext was part of the + * key nothing could ever hit, so a stale entry was unreachable by construction; now that + * the cache works, writes have to publish themselves. + * + * A single delete here is not quite enough: a reader that fetched the pre-write value from + * the provider a moment earlier can still complete its own cache write AFTER this delete + * finishes, silently reintroducing the stale entry for the rest of the TTL -- nothing else + * would clear it until the next write. Scheduling a second delete closes that window the + * conventional way: any straggler write that lands in the gap gets cleared shortly after, + * long before an operator or caller would reasonably treat it as current. + */ + private[util] def invalidateEndpointMappingCache(): Unit = { + Redis.deleteKeysByPattern("*getEndpointMappings*") + code.actorsystem.ObpActorSystem.localActorSystem.scheduler.scheduleOnce( + endpointMappingCacheInvalidationDelay + )(Redis.deleteKeysByPattern("*getEndpointMappings*"))( + code.actorsystem.ObpActorSystem.localActorSystem.dispatcher + ) + () + } + + private[util] val endpointMappingCacheInvalidationDelay: scala.concurrent.duration.FiniteDuration = + scala.concurrent.duration.FiniteDuration( + APIUtil.getPropsAsIntValue("endpointMapping.cache.invalidation.delay.ms", 500), "ms") + def getEndpointMappingById(bankId: Option[String], endpointMappingId : String, callContext: Option[CallContext]): OBPReturnType[EndpointMappingT] = { validateBankId(bankId, callContext) @@ -3378,18 +3416,41 @@ object NewStyle extends MdcLoggable{ private[this] val endpointMappingTTL = APIUtil.getPropsValue(s"endpointMapping.cache.ttl.seconds", "0").toInt - def getEndpointMappings(bankId: Option[String], callContext: Option[CallContext]): OBPReturnType[List[EndpointMappingT]] = Future{ + /** + * The memoized half of getEndpointMappings, split into its own method for two reasons. + * + * Neither the key nor the cached value may mention the callContext. The key, because + * CacheKeyFromArguments renders every un-annotated parameter and CallContext carries + * per-request state (startTime, correlationId, url, verb, ipAddress, user) - keying on it + * made the key unique per request, so the cache could never hit. The value, because a hit + * would hand the caller the originating request's CallContext, and because chill/Kryo + * cannot encode the lambda reachable through CallContext.resourceDocument: every write of + * the old (mappings, callContext) tuple failed and cachePut swallowed it as "result served + * uncached", so endpointMapping.cache.ttl.seconds bought nothing but a WARN per call. + * + * A parameter-less signature rather than `@CacheKeyOmit callContext` on the caller, because + * CacheKeyFromArguments reads the parameters of the method whose body ENDS in buildCacheKey. + * Binding the result to a val first (`val x = buildCacheKey {...}; (x, callContext)`) leaves + * the macro with no parameters to render and it emits `Nil.mkString("_")` - an empty + * argument segment, i.e. every bankId sharing one entry. Keep buildCacheKey as the tail + * expression here; `NewStyle.function.getEndpointMappings` is verified by javap to render + * `bankId :: Nil`. + */ + private def getEndpointMappingsCached(bankId: Option[String]): List[EndpointMappingT] = { import scala.concurrent.duration._ - validateBankId(bankId, callContext) - var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString) CacheKeyFromArguments.buildCacheKey { Caching.memoizeSyncWithProvider(Some(cacheKey.toString()))(endpointMappingTTL.second) { - {(EndpointMappingProvider.endpointMappingProvider.vend.getAllEndpointMappings(bankId), callContext)} + EndpointMappingProvider.endpointMappingProvider.vend.getAllEndpointMappings(bankId) } } } + + def getEndpointMappings(bankId: Option[String], callContext: Option[CallContext]): OBPReturnType[List[EndpointMappingT]] = Future{ + validateBankId(bankId, callContext) + (getEndpointMappingsCached(bankId), callContext) + } /** * Invalidate the Redis-backed resource-doc caches whose contents include diff --git a/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala b/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala new file mode 100644 index 0000000000..641c02ee48 --- /dev/null +++ b/obp-api/src/main/scala/code/api/util/ResourceDocRegistry.scala @@ -0,0 +1,162 @@ +package code.api.util + +import code.api.berlin.group.ConstantsBG +import code.api.util.APIUtil.ResourceDoc +import com.openbankproject.commons.util.ApiVersion._ +import com.openbankproject.commons.util.{ApiStandards, ApiVersion, ScannedApiVersion} + +import scala.collection.immutable.ListMap + +/** + * Single source of truth for "which resource docs does version X serve" — used both by the + * per-version resource-docs dispatcher (ResourceDocsAPIMethods.getResourceDocsList, i.e. + * /resource-docs/{VERSION}/... and API Explorer) and by the global operation-id union + * (allStaticResourceDocs / getAllResourceDocs, used wherever an operation id must be resolved: + * api-collection-endpoint creation, top-apis/popular-apis lookups, metrics, ...). + * + * These used to be two independently hand-maintained registries and drifted three times: Berlin + * Group v2 was served by the dispatcher but missing from the union (BGv2-getAccountDetails could + * not be added to an API collection), the union was based on the v6 aggregation excluding v7-only + * operation ids, and the Berlin Group v1.3 alias was missing from the union too. Deriving both + * from one `registry` map makes that class of drift structurally impossible: add a version once, + * both call sites see it. + * + * Rule for adding a new API standard: implement `with ScannedApis` (see that trait) and it is + * picked up automatically via the `scanned` half of `registry` below — no edit needed here. Only + * standards that cannot be discovered that way (or that need to override the source composing + * function, e.g. the cumulative per-version OBP-standard aggregations) need an `explicit` entry. + * + * Deliberately its own file/object, NOT a member of `APIUtil`: the Implementations* objects for + * each version re-enter `APIUtil` during their own initialization (prop lookups, etc.), so a + * strict `val` living inside `APIUtil` risks a class-init deadlock. Everything here stays `lazy` + * and is first touched at request/test time, well after Props and `ApiVersion.setUrlPrefix` have + * run in Boot. + */ +object ResourceDocRegistry { + + /** version -> that surface's docs. Thunks, not values: the ScannedApis-discovered arms are + * lazy vals themselves and the OBP-standard aggregations are cumulative lazy vals too — wrapping + * in a function defers evaluation to first use of THIS registry, not construction of the map. */ + lazy val registry: ListMap[ApiVersion, () => Seq[ResourceDoc]] = { + val explicit: ListMap[ApiVersion, () => Seq[ResourceDoc]] = ListMap( + v7_0_0 -> (() => code.api.v7_0_0.Http4s700.allResourceDocs.toSeq), + v6_0_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v600.toSeq), + v5_1_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v510.toSeq), + v5_0_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v500.toSeq), + v4_0_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v400.toSeq), + v3_1_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v310.toSeq), + v3_0_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v300.toSeq), + v2_2_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v220.toSeq), + v2_1_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v210.toSeq), + v2_0_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v200.toSeq), + v1_4_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v140.toSeq), + v1_3_0 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v130.toSeq), + v1_2_1 -> (() => code.api.util.http4s.Http4sResourceDocAggregation.v121.toSeq), + `dynamic-endpoint` -> (() => code.api.dynamic.endpoint.OBPAPIDynamicEndpoint.allResourceDocs.toSeq), + `dynamic-entity` -> (() => code.api.dynamic.entity.OBPAPIDynamicEntity.allResourceDocs.toSeq) + // Neither Berlin Group nor UK Open Banking is listed here: they are all ScannedApis + // registrants, so `scanned` picks them up and -- crucially -- orders them against each other + // by standardPrecedence below. Naming one of them here would pin it ahead of that ordering. + ) + // Every standard discovered via ScannedApis (UK OB 200/310/401, BG v1.3 canonical + alias, + // BG v2, and any future `with ScannedApis` standard), folded into a ListMap so the registry has + // ONE defined iteration order. + // + // Order matters beyond determinism: Http4s600's top-apis/popular-apis and JSONFactory6.0.0's + // metrics build `partialFunctionName -> operationId` with `.toMap`, where the LAST entry wins. + // Berlin Group and UK Open Banking share three partialFunctionNames -- getBalances, + // getAccountList, getAccountBalances -- and the hand-written union this registry replaced + // listed UK before BG, so Berlin Group won all three. Sorting alphabetically put UK last and + // silently flipped them to UKv4.0.1-getBalances / UKv2.0-getAccountList / + // UKv2.0-getAccountBalances in metrics output, so the precedence is now explicit. + val scanned: ListMap[ApiVersion, () => Seq[ResourceDoc]] = + ScannedApis.versionMapScannedApis.toSeq + .collect { case (version: ScannedApiVersion, apis) if !explicit.contains(version) => + version -> (() => apis.allResourceDocs.toSeq) } + .sortBy(entry => sortKey(code.api.berlin.group.v1_3.OBP_BERLIN_GROUP_1_3_Alias.apiVersion)(entry._1)) + .foldLeft(ListMap.empty[ApiVersion, () => Seq[ResourceDoc]])(_ + _) + explicit ++ scanned + } + + /** + * Standards in ASCENDING precedence: a standard later in this list wins a partialFunctionName it + * shares with an earlier one, because the `.toMap` consumers keep the last entry. This + * reproduces the order of the hand-written union that preceded this registry (UK Open Banking, + * then Berlin Group). A standard that is not listed ranks below all of them. + */ + private val standardPrecedence: List[String] = + List(ApiVersion.ukOpenBankingV20.apiStandard, ConstantsBG.berlinGroupVersion1.apiStandard) + + /** Below every entry of standardPrecedence, whose lowest index is -1 for an unlisted standard. */ + private val derivedStandardRank: Int = -2 + + /** + * Total order over registry keys: precedence first, then the version's own identity. + * + * `derivedAliasVersion` is the version of a standard that merely re-publishes another standard's + * docs -- today only the Berlin Group v1.3 alias. It is matched by identity, NOT by its + * apiStandard, because that string is the first segment of `berlin_group_v1_3_alias_path` and a + * deployment may legitimately choose one that an existing standard already uses: configured as + * "BG/v9" the alias would otherwise rank alongside Berlin Group and, sorting after "v2", let its + * re-stamped copies win getBalances, getAccountList and getAccountBalances away from the + * canonical docs it copied. Ranking it derivedStandardRank keeps that impossible for any + * configuration. + * + * The tie-breaker is (apiStandard, apiShortVersion) rather than fullyQualifiedVersion because + * that pair is exactly ScannedApiVersion's equals/hashCode key, so two distinct keys of a Map + * keyed by version always differ in it and sortBy never has to fall back to the unordered input. + * fullyQualifiedVersion concatenates the two (apiStandard.toUpperCase + apiShortVersion) and can + * therefore collide across distinct keys -- ("BG", "v1.3") and ("BGV", "1.3") both render + * "BGV1.3" -- which a deployment could reach through berlin_group_v1_3_alias_path. + * + * Curried and package-private so a test can rank against a synthetic alias without having to + * restart the JVM under a different berlin_group_v1_3_alias_path. + */ + private[util] def sortKey(derivedAliasVersion: ScannedApiVersion) + (version: ScannedApiVersion): (Int, String, String) = { + val rank = + if (version == derivedAliasVersion) derivedStandardRank + else standardPrecedence.indexOf(version.apiStandard) + (rank, version.apiStandard, version.apiShortVersion) + } + + /** What the per-version resource-docs dispatcher serves for this version (empty if unknown). */ + def docsFor(version: ApiVersion): Seq[ResourceDoc] = registry.get(version).map(_ ()).getOrElse(Nil) + + /** + * The OBP-standard surface the global union is built from. + * + * The registry also holds the cumulative aggregations for every older OBP version, because the + * dispatcher must still serve /resource-docs/OBPv4.0.0/obp and friends. Those are NOT folded into + * the union: they are not subsets of the v7 aggregation (an endpoint dropped after v4 keeps its + * operation id there), so including them would add ~287 operation ids that the union never + * carried, 234 of which collide on partialFunctionName with an entry already present -- and the + * `.toMap` consumers above would then report the OLDEST id (getBanks -> OBPv1.2.1-getBanks) + * instead of the current one in metrics, top-apis and popular-apis output. + * + * Consequence, deliberately accepted: an operation id that exists ONLY in a superseded + * aggregation stays unresolvable by api-collection-endpoint creation, exactly as before this + * refactor. ResourceDocRegistryParityTest pins that this constant is the newest OBP-standard + * version in the registry, so adding v8 without moving it fails the build rather than silently + * dropping v8-only operation ids from the union. + */ + val obpUnionVersion: ApiVersion = v7_0_0 + + private def isObpStandard(version: ApiVersion): Boolean = version match { + case sv: ScannedApiVersion => sv.apiStandard == ApiStandards.obp.toString + case _ => false + } + + /** Versions whose docs make up the global union: the current OBP surface plus every non-OBP + * standard. Excluding the other OBP-standard keys also excludes `dynamic-endpoint` / + * `dynamic-entity` (both carry apiStandard "obp"), which must stay out for a second reason: + * they are runtime-mutable and APIUtil.getAllResourceDocs appends them FRESH on every call, so + * caching them in this lazy union would serve stale dynamic docs. */ + lazy val unionVersions: Seq[ApiVersion] = + registry.keys.filter(v => v == obpUnionVersion || !isObpStandard(v)).toSeq + + /** The global operation-id union. Deduped by operationId: the surfaces legitimately overlap, and + * consumers only ever `.find`/build a lookup map from this list, never rely on duplicates. */ + lazy val allStaticResourceDocs: List[ResourceDoc] = + unionVersions.flatMap(docsFor).toList.distinctBy(_.operationId) +} diff --git a/obp-api/src/main/scala/code/api/util/ScannedApis.scala b/obp-api/src/main/scala/code/api/util/ScannedApis.scala index bfbc688985..238de80b0e 100644 --- a/obp-api/src/main/scala/code/api/util/ScannedApis.scala +++ b/obp-api/src/main/scala/code/api/util/ScannedApis.scala @@ -21,9 +21,21 @@ trait ScannedApis { object ScannedApis { /** * this map value are all scanned objects those extends ScannedApiVersion, the key is it apiVersion field + * + * Registrants whose version carries no urlPrefix, apiStandard and apiShortVersion are dropped: + * such a version addresses nothing, and it is how a configuration-gated standard reports itself + * as switched off (OBP_BERLIN_GROUP_1_3_Alias falls back to ScannedApiVersion("", "", "") when + * berlin_group_v1_3_alias_path is unset). Keeping it here leaked into everything built from this + * map: its fullyQualifiedVersion is "" too, so ApiVersionUtils.valueOf("") resolved successfully + * and GET /obp/v7.0.0/resource-docs//obp answered 200 with an empty document list instead of the + * 400 InvalidApiVersionString any other unknown version string gets. */ lazy val versionMapScannedApis: Map[ScannedApiVersion, ScannedApis] = ClassScanUtils.getSubTypeObjects[ScannedApis] + .filter(it => isAddressable(it.apiVersion)) .map(it=> (it.apiVersion, it)) .toMap + + private def isAddressable(version: ScannedApiVersion): Boolean = + version.urlPrefix.trim.nonEmpty || version.apiStandard.trim.nonEmpty || version.apiShortVersion.trim.nonEmpty } diff --git a/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala b/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala index 8e7768e92a..5af1eb65ae 100644 --- a/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala +++ b/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala @@ -45,7 +45,8 @@ object WriteMetricUtil extends MdcLoggable { duration: Long, responseBodyToWrite: String, sourceIp: String, - targetIp: String) + targetIp: String, + authType: String) private def persistAndPublishMetric(responseBody: Any, cc: CallContextLight): Unit = { val fields = MetricFields( @@ -58,7 +59,8 @@ object WriteMetricUtil extends MdcLoggable { duration = callDuration(cc), responseBodyToWrite = responseBodyForMetric(responseBody, cc), sourceIp = requestHeaderValue(cc, "x-forwarded-for"), - targetIp = requestHeaderValue(cc, "x-forwarded-host") + targetIp = requestHeaderValue(cc, "x-forwarded-host"), + authType = deriveAuthType(cc) ) // enqueue synchronously so flush() in tests reliably drains this metric before assertions @@ -74,6 +76,30 @@ object WriteMetricUtil extends MdcLoggable { } } + /** + * Authentication SCHEME of the call — never the credential itself. "Consent" wins + * outright: when a consent authenticated the call, the Authorization header (if any) + * was not what authorized it. The rest is read off the Authorization header shape, + * with the gateway payload / direct-login params as fallbacks for flows that + * populate those without a header. + */ + private[util] def deriveAuthType(cc: CallContextLight): String = { + if (cc.consentReferenceId.isDefined) "Consent" + else cc.authReqHeaderField.map(_.trim) match { + case Some(h) if h.startsWith("DirectLogin") => "DirectLogin" + case Some(h) if h.startsWith("Bearer") => "OAuth2" + case Some(h) if h.startsWith("GatewayLogin") => "GatewayLogin" + case Some(h) if h.startsWith("DAuth") => "DAuth" + case Some(h) if h.startsWith("OAuth") => "OAuth1" + case Some(_) => "Other" + case None => + if (cc.gatewayLoginRequestPayload.isDefined) "GatewayLogin" + else if (cc.directLoginToken != null && cc.directLoginToken.nonEmpty) "DirectLogin" + else if (cc.userId.isDefined) "Other" + else "Anonymous" + } + } + private def callDuration(cc: CallContextLight): Long = (cc.startTime, cc.endTime) match { case (Some(s), Some(e)) => e.getTime - s.getTime @@ -116,7 +142,8 @@ object WriteMetricUtil extends MdcLoggable { code.api.Constant.ApiInstanceId, cc.consentReferenceId.orNull, cc.certificateTrust.orNull, - cc.certificateTrustDetail.orNull + cc.certificateTrustDetail.orNull, + authType ) } catch { case NonFatal(e) => diff --git a/obp-api/src/main/scala/code/api/util/http4s/IdempotencyMiddleware.scala b/obp-api/src/main/scala/code/api/util/http4s/IdempotencyMiddleware.scala index f9e9c5032d..d3d0429de2 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/IdempotencyMiddleware.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/IdempotencyMiddleware.scala @@ -31,11 +31,14 @@ import java.util.Base64 * - Concurrent replay while the original is still in flight: 409 Conflict. * - 5xx responses are NOT cached; clients can retry. * - * Scope: the key is namespaced by SHA-256 of the consumer id, or — when - * unauthenticated — the Authorization header. This prevents key reuse across - * consumers. + * Scope: the key is namespaced by SHA-256 of (consumer id, or — when + * unauthenticated — the Authorization header) AND the resolved operation id. This prevents key + * reuse across consumers AND across endpoints: a client accidentally or deliberately reusing one + * Idempotency-Key against two different operations gets two independent dedup slots instead of + * the second operation being treated as a replay of the first and never executed. * - * Validation: 8..255 printable-ASCII characters. Anything else → 400. + * Validation: 8..255 printable-ASCII characters. Anything else → 400, regardless of whether this + * tier serves the path -- a malformed header is a client error no matter where it resolves. * * Storage: Redis via the existing JedisPool. Two keys per request: * - idem:lock:: → "1" (60s TTL, set with NX) @@ -44,9 +47,32 @@ import java.util.Base64 * Resilience: any Redis error is logged and the request is allowed to proceed * unchanged — the middleware never blocks traffic on cache outages. * - * The middleware should be installed INSIDE ResourceDocMiddleware so the - * CallContext (and therefore the consumer id) is populated before the scope - * key is computed. + * ── Where it may be installed ── + * + * INSIDE ResourceDocMiddleware, on every version's route tree. Both halves matter. + * + * Inside, because the scope key and the request-body hash both come from the CallContext, and + * ResourceDocMiddleware is what populates it. Mounted outside, there is no CallContext at all -- + * no operationId and no httpBody -- so `matchedThisTier` is false for every request and this + * middleware does nothing at all rather than mishandling anything: no lock, no cached response, + * no conflict detection. That is a silent loss of protection, not a silent corruption, but it is + * still a config-only guard against a real risk: on a payment endpoint, dedup quietly not running + * is the difference between "your retry was deduplicated" and "your retry submitted the payment + * again". + * + * On every tree, because Http4sApp composes the versions with `.orElse` and a tree signals "not + * mine" by returning OptionT.none. This middleware therefore has to pass a miss through + * unchanged; an earlier version answered 404 in that case, which terminated the chain -- measured + * on POST /obp/v3.1.0/management/method_routings, which answered 201 without an Idempotency-Key + * and 404 with one. A later version preserved the miss but still spent a full lock-acquire/release + * cycle doing it, using a method+path fallback scope for any tier that had not matched -- which + * meant two requests racing that SAME miss-tier's fallback lock (two genuinely different + * endpoints whose method+path fallback happened to differ less often than intended, or two + * concurrent copies of the request destined for a LATER tier) could have the loser answered a + * definite 409 by a tier that was never going to serve either of them, before the request ever + * reached the tier that would have handled it correctly. `matchedThisTier` closes that: only the + * one tier whose ResourceDocMatcher actually matched (the only tier where operationId is ever + * set) does any lock/response-key work at all. IdempotencyMiddlewareTest pins all of these. */ object IdempotencyMiddleware extends MdcLoggable { @@ -81,14 +107,41 @@ object IdempotencyMiddleware extends MdcLoggable { } else { val key = keyOpt.get if (!isValidKey(key)) { + // A malformed header is a client error whatever the path resolves to, so this + // one deliberately does NOT fall through. OptionT.liftF(invalidKeyResponse(key)) + } else if (!matchedThisTier(req)) { + // operationId is set on the CallContext ONLY by ResourceDocMiddleware.attachToCallContext, + // and only for the one version tree whose ResourceDocMatcher actually matched this + // request (see ResourceDocMiddleware.apply: the `case None` branch never attaches one). + // Every other tree in the `.orElse` chain is about to answer a miss regardless of what + // this middleware does, so doing lock/response-key work there is worse than wasted: two + // requests that legitimately belong to two DIFFERENT trees, or two genuinely concurrent + // copies of the same request, can race the SAME miss-tier's method+path-fallback lock + // and the loser gets a definite 409 here -- terminating the `.orElse` chain on behalf of + // a tree that was never going to serve either of them, before the request ever reaches + // the tree that would have handled it correctly. Skipping straight through at a + // miss-tier removes both the wasted Redis round trips and this false-conflict window; + // only the matching tier ever computes a scope key that means anything durable, so only + // it needs the protection. + routes.run(req) } else { val scope = scopeFor(req) val bodyHash = sha256Hex(bodyFromCallContextOrEmpty(req)) val responseKey = ResponseKeyPrefix + scope + ":" + key val lockKey = LockKeyPrefix + scope + ":" + key - OptionT.liftF(handle(req, routes, responseKey, lockKey, bodyHash)) + // OptionT, not OptionT.liftF: a route MISS has to stay a miss. + // + // Every version's routes are one link in a fallthrough chain -- Http4sApp composes them + // with `.orElse`, and `OptionT.none` is how a tree says "not mine, try the next one". + // Wrapping with liftF made this middleware answer 404 on behalf of a tree that simply + // did not serve the path, which terminated the chain: measured on + // `POST /obp/v3.1.0/management/method_routings`, the request answered 201 without an + // Idempotency-Key and 404 with one, because the first tree it passed through swallowed + // the miss. So the middleware could only ever be installed on the last link. Preserving + // the miss is what makes it safe to install on all of them. + OptionT(handle(req, routes, responseKey, lockKey, bodyHash)) } } } @@ -99,16 +152,16 @@ object IdempotencyMiddleware extends MdcLoggable { responseKey: String, lockKey: String, requestBodyHash: String - ): IO[Response[IO]] = { + ): IO[Option[Response[IO]]] = { IO.blocking(readResponseKey(responseKey)).attempt.flatMap { case Right(Some(envelope)) => if (envelope.requestBodyHash == requestBodyHash) { - IO.pure(rebuildResponse(envelope, replay = true)) + IO.pure(Some(rebuildResponse(envelope, replay = true))) } else { conflictResponse( "Idempotency-Key replayed with a different request body. " + "Use a fresh key for a different request." - ) + ).map(Some(_)) } case Right(None) => @@ -118,7 +171,7 @@ object IdempotencyMiddleware extends MdcLoggable { case Right(false) => conflictResponse( "Idempotent operation already in flight for this Idempotency-Key." - ) + ).map(Some(_)) case Left(t) => logger.warn(s"Idempotency lock unavailable (Redis): ${t.getMessage}") runRoutes(req, routes) @@ -136,41 +189,48 @@ object IdempotencyMiddleware extends MdcLoggable { responseKey: String, lockKey: String, requestBodyHash: String - ): IO[Response[IO]] = { - runRoutes(req, routes).flatMap { resp => - // Drain body so we can both cache and re-emit it. - resp.body.compile.toVector.flatMap { vec => - val bodyBytes = vec.toArray - val rebuilt = resp.withBodyStream(fs2.Stream.emits(bodyBytes).covary[IO]) - - val storeOrReleaseLock: IO[Unit] = - if (resp.status.code >= 500) { - // Don't cache transient failures; release the lock so client can retry. - IO.blocking(deleteKey(lockKey)).attempt.map(_ => ()) - } else { - val envelope = Envelope( - status = resp.status.code, - contentType = resp.headers.get(CIString("Content-Type")).map(_.head.value), - bodyB64 = Base64.getEncoder.encodeToString(bodyBytes), - requestBodyHash = requestBodyHash - ) - IO.blocking { - writeResponseKey(responseKey, envelope) - deleteKey(lockKey) - }.attempt.map { e => - e.left.foreach(t => - logger.warn(s"Failed to cache idempotent response: ${t.getMessage}") + ): IO[Option[Response[IO]]] = { + runRoutes(req, routes).flatMap { + // The lock was taken before the routes ran, so a miss has to give it back -- otherwise a + // path this tree does not serve would hold the key locked for its full 60s TTL and a + // genuine request carrying that key would be refused with 409. + case None => IO.blocking(deleteKey(lockKey)).attempt.as(None) + case Some(resp) => + // Drain body so we can both cache and re-emit it. + resp.body.compile.toVector.flatMap { vec => + val bodyBytes = vec.toArray + val rebuilt = resp.withBodyStream(fs2.Stream.emits(bodyBytes).covary[IO]) + + val storeOrReleaseLock: IO[Unit] = + if (resp.status.code >= 500) { + // Don't cache transient failures; release the lock so client can retry. + IO.blocking(deleteKey(lockKey)).attempt.map(_ => ()) + } else { + val envelope = Envelope( + status = resp.status.code, + contentType = resp.headers.get(CIString("Content-Type")).map(_.head.value), + bodyB64 = Base64.getEncoder.encodeToString(bodyBytes), + requestBodyHash = requestBodyHash ) - () + IO.blocking { + writeResponseKey(responseKey, envelope) + deleteKey(lockKey) + }.attempt.map { e => + e.left.foreach(t => + logger.warn(s"Failed to cache idempotent response: ${t.getMessage}") + ) + () + } } - } - storeOrReleaseLock.as(rebuilt) - } + storeOrReleaseLock.as(Some(rebuilt)) + } } } - private def runRoutes(req: Request[IO], routes: HttpRoutes[IO]): IO[Response[IO]] = - routes.run(req).getOrElseF(IO.pure(Response[IO](Status.NotFound))) + // `.value`, not `getOrElseF(404)` -- see the comment on the OptionT in `apply`. Converting a + // miss into a 404 here is what terminated the version fallthrough chain. + private def runRoutes(req: Request[IO], routes: HttpRoutes[IO]): IO[Option[Response[IO]]] = + routes.run(req).value // ── Validation ───────────────────────────────────────────────────────── @@ -179,16 +239,34 @@ object IdempotencyMiddleware extends MdcLoggable { key.length <= MaxKeyLength && key.forall(c => c >= 0x21 && c <= 0x7E) + // True only for the one version tree whose ResourceDocMatcher matched this request -- + // ResourceDocMiddleware.attachToCallContext is the sole place operationId is ever set, and it + // runs only on a match (see ResourceDocMiddleware.apply's `case Some(resourceDoc)` branch; the + // `case None` branch attaches a CallContext with no operationId). Every other tree in the + // `.orElse` chain sees operationId absent here and skips idempotency handling entirely, because + // it is about to answer a miss regardless of what this middleware does. + private def matchedThisTier(req: Request[IO]): Boolean = + req.attributes.lookup(Http4sRequestAttributes.callContextKey).exists(_.operationId.isDefined) + // ── Scope ────────────────────────────────────────────────────────────── private def scopeFor(req: Request[IO]): String = { val ccOpt = req.attributes.lookup(Http4sRequestAttributes.callContextKey) - val raw = ccOpt + val consumerOrAuth = ccOpt .flatMap(_.consumer.map(_.consumerId.get).toOption) .filter(_.nonEmpty) .orElse(req.headers.get(AuthorizationHeader).map(_.head.value)) .getOrElse("anonymous") - sha256Hex(raw).take(16) + // operationId is the canonical identity of "which endpoint" -- set by ResourceDocMiddleware + // once it has matched a ResourceDoc, and stable across path-template placeholders and + // bridge-cascade path rewrites (v400->v310->...). scopeFor only ever runs after + // matchedThisTier has confirmed operationId is present, so the method+path fallback below is + // purely defensive -- it should never actually be exercised in production, only under a + // future bug that calls scopeFor without that guard. + val endpoint = ccOpt + .flatMap(_.operationId) + .getOrElse(s"${req.method.name} ${req.uri.path.renderString}") + sha256Hex(s"$consumerOrAuth|$endpoint").take(16) } // ── Body hash ────────────────────────────────────────────────────────── diff --git a/obp-api/src/main/scala/code/api/util/migration/Migration.scala b/obp-api/src/main/scala/code/api/util/migration/Migration.scala index 47eb4c711b..9a094352ef 100644 --- a/obp-api/src/main/scala/code/api/util/migration/Migration.scala +++ b/obp-api/src/main/scala/code/api/util/migration/Migration.scala @@ -157,6 +157,7 @@ object Migration extends MdcLoggable { migrateChatRoomCreatedByAndLastMessageSender() migrateConsentReferenceIdToUuid(startedBeforeSchemifier) migrateMetricConsentReferenceId(startedBeforeSchemifier) + migrateMetricAuthType(startedBeforeSchemifier) migrateMetricCertificateTrust(startedBeforeSchemifier) dropFastFirehoseAccountsViews(startedBeforeSchemifier) alterDynamicResourceDocBodyFieldsLength() @@ -808,6 +809,18 @@ object Migration extends MdcLoggable { } } + private def migrateMetricAuthType(startedBeforeSchemifier: Boolean): Boolean = { + if(startedBeforeSchemifier == true) { + logger.warn(s"Migration.database.migrateMetricAuthType(true) cannot be run before Schemifier.") + true + } else { + val name = nameOf(migrateMetricAuthType(startedBeforeSchemifier)) + runOnce(name) { + MigrationOfMetricAuthType.migrate(name) + } + } + } + private def migrateMetricCertificateTrust(startedBeforeSchemifier: Boolean): Boolean = { if(startedBeforeSchemifier == true) { logger.warn(s"Migration.database.migrateMetricCertificateTrust(true) cannot be run before Schemifier.") diff --git a/obp-api/src/main/scala/code/api/util/migration/MigrationOfActivityDashboardIndexes.scala b/obp-api/src/main/scala/code/api/util/migration/MigrationOfActivityDashboardIndexes.scala index 8716a35c38..278565b0a9 100644 --- a/obp-api/src/main/scala/code/api/util/migration/MigrationOfActivityDashboardIndexes.scala +++ b/obp-api/src/main/scala/code/api/util/migration/MigrationOfActivityDashboardIndexes.scala @@ -88,7 +88,7 @@ object MigrationOfActivityDashboardIndexes { * Index on resourceuser.createdbyconsentid. * * The delegation registry: consent-agent fan-down (/my/metrics, /my/banks) and - * CallContext.effectiveHumanUserId look up agent users by the consent that minted them. + * CallContext.accountableUserId look up agent users by the consent that minted them. * Unindexed this is a full scan of resourceuser on every such request, which matters on * consent-heavy instances where every consent mints a user row. */ @@ -130,7 +130,7 @@ object MigrationOfActivityDashboardIndexes { s"""Added index on resourceuser.createdbyconsentid |Executed SQL: |$executedSql - |Serves the consent-agent delegation fan-down (/my/metrics, /my/banks, effectiveHumanUserId). + |Serves the consent-agent delegation fan-down (/my/metrics, /my/banks, accountableUserId). |""".stripMargin isSuccessful = true saveLog(name, commitId, isSuccessful, startDate, endDate, comment) diff --git a/obp-api/src/main/scala/code/api/util/migration/MigrationOfMetricAuthType.scala b/obp-api/src/main/scala/code/api/util/migration/MigrationOfMetricAuthType.scala new file mode 100644 index 0000000000..c79b85ccfe --- /dev/null +++ b/obp-api/src/main/scala/code/api/util/migration/MigrationOfMetricAuthType.scala @@ -0,0 +1,74 @@ +package code.api.util.migration + +import code.api.util.APIUtil +import code.api.util.migration.Migration.{DbFunction, saveLog} +import code.metrics.MappedMetric +import net.liftweb.mapper.Schemifier + +/** + * Migration: add `auth_type VARCHAR(32)` to both the live `Metric` table and the + * `metricarchive` table — the authentication SCHEME of each call ("Consent", + * "OAuth2", "OAuth1", "DirectLogin", "GatewayLogin", "DAuth", "Anonymous", + * "Other"), never the credential itself. + * + * No backup and no backfill: the column is additive and nullable — historical rows + * legitimately predate it and stay null. No index: always queried alongside the + * indexed date range. + * + * Lift's Schemifier auto-creates the column on fresh deploys from the updated model; + * this migration handles existing deploys. Table name note as in + * MigrationOfMetricConsentReferenceId: unquoted lowercase `metric` everywhere. + */ +object MigrationOfMetricAuthType { + + def migrate(name: String): Boolean = { + DbFunction.tableExists(MappedMetric) match { + case true => + val startDate = System.currentTimeMillis() + val commitId: String = APIUtil.gitCommit + val dbDriver = APIUtil.getPropsValue("db.driver") openOr "org.h2.Driver" + val isMssql = dbDriver.contains("com.microsoft.sqlserver.jdbc.SQLServerDriver") + var isSuccessful = false + val sqlLog = new StringBuilder() + + try { + val addColumnMetric = if (isMssql) { + "ALTER TABLE metric ADD auth_type VARCHAR(32) NULL;" + } else { + "ALTER TABLE metric ADD COLUMN IF NOT EXISTS auth_type VARCHAR(32);" + } + sqlLog.append(DbFunction.maybeWrite(true, Schemifier.infoF _)(() => addColumnMetric)).append("\n") + + val addColumnArchive = if (isMssql) { + "ALTER TABLE metricarchive ADD auth_type VARCHAR(32) NULL;" + } else { + "ALTER TABLE metricarchive ADD COLUMN IF NOT EXISTS auth_type VARCHAR(32);" + } + sqlLog.append(DbFunction.maybeWrite(true, Schemifier.infoF _)(() => addColumnArchive)).append("\n") + + isSuccessful = true + } catch { + case e: Exception => + isSuccessful = false + sqlLog.append(s"\nException: ${e.getMessage}\n") + } + + val endDate = System.currentTimeMillis() + val comment: String = + s"""Executed SQL: + |$sqlLog + |""".stripMargin + saveLog(name, commitId, isSuccessful, startDate, endDate, comment) + isSuccessful + + case false => + val startDate = System.currentTimeMillis() + val commitId: String = APIUtil.gitCommit + val isSuccessful = false + val endDate = System.currentTimeMillis() + val comment: String = s"""${MappedMetric._dbTableNameLC} table does not exist""".stripMargin + saveLog(name, commitId, isSuccessful, startDate, endDate, comment) + isSuccessful + } + } +} diff --git a/obp-api/src/main/scala/code/api/v1_2_1/Http4s121.scala b/obp-api/src/main/scala/code/api/v1_2_1/Http4s121.scala index 3d7ceb4127..891231fd92 100644 --- a/obp-api/src/main/scala/code/api/v1_2_1/Http4s121.scala +++ b/obp-api/src/main/scala/code/api/v1_2_1/Http4s121.scala @@ -10,6 +10,7 @@ import code.api.util.ApiTag._ import code.api.util.ErrorMessages._ import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps, callContextKey} import code.api.util.http4s.Http4sCallContextBuilder +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.http4s.ResourceDocMiddleware import code.api.util.newstyle.ViewNewStyle import code.api.util.{CallContext, CustomJsonFormats, NewStyle} @@ -2669,7 +2670,7 @@ object Http4s121 { } val allRoutesWithMiddleware: HttpRoutes[IO] = { - val middlewareWrapped = ResourceDocMiddleware.apply(resourceDocs)(allRoutes) + val middlewareWrapped = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes)) // bankById runs before middleware so it can return 400 (not 404) for unknown bank Kleisli[HttpF, Request[IO], Response[IO]] { req => bankById.run(req).orElse(middlewareWrapped.run(req)) diff --git a/obp-api/src/main/scala/code/api/v1_3_0/Http4s130.scala b/obp-api/src/main/scala/code/api/v1_3_0/Http4s130.scala index c3fe407a9e..27d0ac3d82 100644 --- a/obp-api/src/main/scala/code/api/v1_3_0/Http4s130.scala +++ b/obp-api/src/main/scala/code/api/v1_3_0/Http4s130.scala @@ -9,6 +9,7 @@ import code.api.util.ApiTag._ import code.api.util.ErrorMessages._ import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.NewStyle import code.api.v1_2_1.JSONFactory import com.github.dwickern.macros.NameOf.nameOf @@ -126,7 +127,7 @@ object Http4s130 { .orElse(getCardsForBank.run(req)) } - val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allOwnRoutes) + val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allOwnRoutes)) // ─── path-rewriting bridge: /obp/v1.3.0/… → /obp/v1.2.1/… ───────────── // Delegates to Http4s121 so all inherited v1.2.1 endpoints are served diff --git a/obp-api/src/main/scala/code/api/v1_4_0/Http4s140.scala b/obp-api/src/main/scala/code/api/v1_4_0/Http4s140.scala index 1f5c17a58a..d9c7a2404d 100644 --- a/obp-api/src/main/scala/code/api/v1_4_0/Http4s140.scala +++ b/obp-api/src/main/scala/code/api/v1_4_0/Http4s140.scala @@ -10,6 +10,7 @@ import code.api.util.ApiTag._ import code.api.util.ErrorMessages._ import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.{APIUtil, NewStyle} import code.api.v1_2_1.{JSONFactory, SuccessMessage} import code.atms.Atms @@ -475,7 +476,7 @@ object Http4s140 { .orElse(addCustomer.run(req)) } - val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allOwnRoutes) + val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allOwnRoutes)) // ─── path-rewriting bridge: /obp/v1.4.0/… → /obp/v1.3.0/… ────────────── // Delegates to Http4s130 so all inherited v1.3.0 and v1.2.1 endpoints are diff --git a/obp-api/src/main/scala/code/api/v2_0_0/Http4s200.scala b/obp-api/src/main/scala/code/api/v2_0_0/Http4s200.scala index 38b4915de1..c93256e6b9 100644 --- a/obp-api/src/main/scala/code/api/v2_0_0/Http4s200.scala +++ b/obp-api/src/main/scala/code/api/v2_0_0/Http4s200.scala @@ -13,6 +13,7 @@ import code.api.util.ApiTag._ import code.api.util.ErrorMessages._ import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.newstyle.ViewNewStyle import code.api.util.{APIUtil, ApiRole, CustomJsonFormats, NewStyle} import code.api.v1_2_1.{JSONFactory => JSONFactory121, SuccessMessage} @@ -845,8 +846,14 @@ object Http4s200 { isValidID(bank.bankId.value) } loggedInUserId = user.userId - userIdAccountOwner = if (body.user_id.nonEmpty) body.user_id else loggedInUserId + // Implicit owner resolves to the HUMAN: under a Consent the caller is the + // per-consent shadow, and an account held by it strands when the consent dies. + userIdAccountOwner = if (body.user_id.nonEmpty) body.user_id else cc.accountableUserId (postedOrLoggedInUser, cc2) <- NewStyle.function.findByUserId(userIdAccountOwner, Some(cc)) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + _ <- code.util.Helper.booleanToFuture( + s"$InvalidUserId user_id names a consent user (an agent identity minted by a Consent). Accounts are held by humans - use the granting user's USER_ID.", + failCode = 400, cc = cc2)(!postedOrLoggedInUser.isConsentUser) _ <- if (userIdAccountOwner == loggedInUserId) Future.successful(Full(())) else code.util.Helper.booleanToFuture( s"${UserHasMissingRoles} $canCreateAccount or create account for self", failCode = 403, cc = Some(cc)) { @@ -1188,7 +1195,13 @@ object Http4s200 { case req @ POST -> `prefixPath` / "users" / userId / "entitlements" => EndpointHelpers.withUserAndBodyCreated[CreateEntitlementJSON, EntitlementJSON](req) { (user, body, cc) => for { - (_, cc2) <- NewStyle.function.findByUserId(userId, Some(cc)) + (targetUser, cc2) <- NewStyle.function.findByUserId(userId, Some(cc)) + // Explicit target: fail loud rather than redirect. A consent user (an agent + // identity minted by a Consent) cannot hold durable roles — grant to the + // granting human instead. + _ <- code.util.Helper.booleanToFuture( + s"$InvalidUserId USER_ID names a consent user (an agent identity minted by a Consent). Entitlements target humans - use the granting user's USER_ID.", + failCode = 400, cc = cc2)(!targetUser.isConsentUser) role <- Future { unboxFullOrFail( net.liftweb.util.Helpers.tryo { ApiRole.valueOf(body.role_name) }, @@ -1586,7 +1599,7 @@ object Http4s200 { .orElse(elasticSearchMetrics.run(req)) } - val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allOwnRoutes) + val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allOwnRoutes)) // ─── path-rewriting bridge: /obp/v2.0.0/… → /obp/v1.4.0/… ────────────── // Delegates to Http4s140 so all inherited v1.4.0/v1.3.0/v1.2.1 endpoints are diff --git a/obp-api/src/main/scala/code/api/v2_1_0/Http4s210.scala b/obp-api/src/main/scala/code/api/v2_1_0/Http4s210.scala index 4081ac34b4..cd31ae8891 100644 --- a/obp-api/src/main/scala/code/api/v2_1_0/Http4s210.scala +++ b/obp-api/src/main/scala/code/api/v2_1_0/Http4s210.scala @@ -12,6 +12,7 @@ import code.api.util.ApiTag._ import code.api.util.ErrorMessages._ import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.newstyle.ViewNewStyle import code.api.util.{APIUtil, ApiRole, CallContext, CustomJsonFormats, NewStyle} import code.api.v1_2_1.{JSONFactory => JSONFactory121, SuccessMessage} @@ -1393,7 +1394,7 @@ object Http4s210 { .orElse(getMetrics.run(req)) } - val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allOwnRoutes) + val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allOwnRoutes)) // ─── path-rewriting bridge: /obp/v2.1.0/… → /obp/v2.0.0/… ────────────── // Delegates to Http4s200 so all inherited v2.0.0/v1.4.0/v1.3.0/v1.2.1 endpoints diff --git a/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala b/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala index f62b53c99a..25f50b29a1 100644 --- a/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala +++ b/obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala @@ -13,6 +13,7 @@ import code.api.util.Glossary import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import java.util.Date import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.newstyle.ViewNewStyle import code.api.util.{APIUtil, CallContext, CustomJsonFormats, NewStyle} import code.api.v1_2_1.{CreateViewJsonV121, JSONFactory => JSONFactory121, UpdateViewJsonV121} @@ -465,17 +466,20 @@ object Http4s220 { bank.swift_bic, bank.national_identifier, bank.bank_routing.scheme, bank.bank_routing.address, Some(cc) ) + // Creator grants target the HUMAN (see v6.0.0 createBank): under a Consent the + // authenticated user is a per-consent shadow, and roles granted to it are stranded. + humanUserId = cc.accountableUserId entitlements <- Future { unboxFullOrFail( - code.entitlement.Entitlement.entitlement.vend.getEntitlementsByUserId(user.userId), + code.entitlement.Entitlement.entitlement.vend.getEntitlementsByUserId(humanUserId), Some(cc), UnknownError) } _ <- Future { val bankEntitlements = entitlements.filter(_.bankId == bank.id) if (!bankEntitlements.exists(_.roleName == canCreateEntitlementAtOneBank.toString())) - code.entitlement.Entitlement.entitlement.vend.addEntitlement(bank.id, user.userId, canCreateEntitlementAtOneBank.toString()) + code.entitlement.Entitlement.entitlement.vend.addEntitlement(bank.id, humanUserId, canCreateEntitlementAtOneBank.toString(), grantedByUserId = Some(user.userId)) if (!bankEntitlements.exists(_.roleName == canReadDynamicResourceDocsAtOneBank.toString())) - code.entitlement.Entitlement.entitlement.vend.addEntitlement(bank.id, user.userId, canReadDynamicResourceDocsAtOneBank.toString()) + code.entitlement.Entitlement.entitlement.vend.addEntitlement(bank.id, humanUserId, canReadDynamicResourceDocsAtOneBank.toString(), grantedByUserId = Some(user.userId)) } } yield JSONFactory220.createBankJSON(success) } @@ -1064,7 +1068,7 @@ object Http4s220 { .orElse(createCounterparty.run(req)) } - val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allOwnRoutes) + val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allOwnRoutes)) // ─── path-rewriting bridge: /obp/v2.2.0/… → /obp/v2.1.0/… ────────────── diff --git a/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala b/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala index 4641e6ab44..4bbe8c6dc7 100644 --- a/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala +++ b/obp-api/src/main/scala/code/api/v3_0_0/Http4s300.scala @@ -15,6 +15,7 @@ import code.api.util.ApiTag._ import code.api.util.ErrorMessages._ import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.newstyle.ViewNewStyle import code.api.util.{APIUtil, CallContext, CustomJsonFormats, NewStyle} import code.api.v1_2_1.JSONFactory @@ -1655,10 +1656,14 @@ object Http4s300 { _ <- code.util.Helper.booleanToFuture( if (ApiRole.valueOf(body.role_name).requiresBankId) EntitlementIsBankRole else EntitlementIsSystemRole, cc = Some(cc)) { ApiRole.valueOf(body.role_name).requiresBankId == body.bank_id.nonEmpty } + // A request for power is a request BY the human: under a Consent the caller is a + // per-consent shadow, and a request filed for it would have an admin granting to + // an identity that dies with the consent (the grant endpoint now rejects that). + requesterUserId = cc.accountableUserId _ <- code.util.Helper.booleanToFuture(EntitlementRequestAlreadyExists, cc = Some(cc)) { - EntitlementRequest.entitlementRequest.vend.getEntitlementRequest(body.bank_id, user.userId, body.role_name).isEmpty + EntitlementRequest.entitlementRequest.vend.getEntitlementRequest(body.bank_id, requesterUserId, body.role_name).isEmpty } - addedEntitlementRequest <- EntitlementRequest.entitlementRequest.vend.addEntitlementRequestFuture(body.bank_id, user.userId, body.role_name) map { + addedEntitlementRequest <- EntitlementRequest.entitlementRequest.vend.addEntitlementRequestFuture(body.bank_id, requesterUserId, body.role_name) map { x => unboxFullOrFail(x, Some(cc), EntitlementRequestCannotBeAdded) } } yield JSONFactory300.createEntitlementRequestJSON(addedEntitlementRequest) @@ -2303,7 +2308,7 @@ object Http4s300 { .orElse(bankById.run(req)) } - val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allOwnRoutes) + val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allOwnRoutes)) // ─── path-rewriting bridge: /obp/v3.0.0/… → /obp/v2.2.0/… ────────────── diff --git a/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala b/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala index 168094cbc3..1d96e56c19 100644 --- a/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala +++ b/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala @@ -17,6 +17,7 @@ import code.api.util.CertificateUtil import code.api.util.{ApiTrigger, Consent, Glossary, SecureRandomUtil} import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.newstyle.{BalanceNewStyle, ViewNewStyle} import code.api.util.{APIUtil, CallContext, CustomJsonFormats, NewStyle, OBPBankId, RateLimitingUtil} import code.api.v1_2_1.{JSONFactory, RateLimiting} @@ -3063,8 +3064,9 @@ object Http4s310 { ) // ─── updateAccountApplicationStatus (PUT) ──────────────────────────────── - // Side effect: when status == "ACCEPTED", a new bank account is created for the - // logged-in user. Preserved verbatim from the Lift implementation. + // Side effect: when status == "ACCEPTED", a new bank account is created and the + // APPLICANT (the application's user) becomes its holder. The Lift implementation + // (and its verbatim port) made the logged-in approver the holder — fixed 2026-09. val updateAccountApplicationStatus: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ PUT -> `prefixPath` / "banks" / _ / "account-applications" / accountApplicationIdStr => @@ -3074,24 +3076,36 @@ object Http4s310 { _ <- NewStyle.function.tryons(s"$InvalidJsonFormat status should not be blank.", 400, Some(cc)) { org.apache.commons.lang3.Validate.notBlank(putJson.status) } - (_, _) <- NewStyle.function.getAccountApplicationById(accountApplicationIdStr, Some(cc)) - (accountApplication, _) <- NewStyle.function.updateAccountApplicationStatus(accountApplicationIdStr, putJson.status, Some(cc)) - userIdOpt = Option(accountApplication.userId) - customerIdOpt = Option(accountApplication.customerId) + (applicationBefore, _) <- NewStyle.function.getAccountApplicationById(accountApplicationIdStr, Some(cc)) + userIdOpt = Option(applicationBefore.userId) + customerIdOpt = Option(applicationBefore.customerId) appUser <- unboxOptionOBPReturnType(userIdOpt.map(NewStyle.function.findByUserId(_, Some(cc)))) customer <- unboxOptionOBPReturnType(customerIdOpt.map(NewStyle.function.getCustomerByCustomerId(_, Some(cc)))) + // Guard BEFORE the status transition commits: failing after it would strand the + // application as ACCEPTED with no account. A consent-user applicant can only come + // from a row that predates the creation-side guard (or was written another way). + _ <- code.util.Helper.booleanToFuture( + s"$InvalidUserId The application's user is a consent user (an agent identity minted by a Consent). Accounts are held by humans - re-apply with the granting user's USER_ID.", + failCode = 400, cc = Some(cc))(!appUser.exists(_.isConsentUser)) + (accountApplication, _) <- NewStyle.function.updateAccountApplicationStatus(accountApplicationIdStr, putJson.status, Some(cc)) _ <- putJson.status match { case "ACCEPTED" => + // The APPLICANT becomes the holder. The Lift-era code (ported verbatim) made the + // approving admin the holder and left appUser unused — every accepted application + // handed the account to whoever clicked approve. Customer-only applications + // (userId empty) keep the legacy approver-as-holder behaviour: there is no user + // to hold, and refusing here would strand the just-committed ACCEPTED status. for { accountId <- Future(AccountId(java.util.UUID.randomUUID().toString)) + holder = appUser.getOrElse(user) (_, _) <- NewStyle.function.createBankAccount( bank.bankId, accountId, accountApplication.productCode.value, "", "EUR", BigDecimal("0"), - user.name, "", + holder.name, "", List.empty, Some(cc)) success <- code.model.dataAccess.BankAccountCreation.setAccountHolderAndRefreshUserAccountAccess( - bank.bankId, accountId, user, Some(cc)) + bank.bankId, accountId, holder, Some(cc)) } yield success case _ => Future("") } @@ -3227,6 +3241,12 @@ object Http4s310 { org.apache.commons.lang3.Validate.isTrue(postedData.user_id.isDefined || postedData.customer_id.isDefined) } appUser <- unboxOptionOBPReturnType(postedData.user_id.map(NewStyle.function.findByUserId(_, Some(cc)))) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + // On ACCEPTED the application's user becomes the account holder, so a consent + // user must be rejected here, before the application is stored. + _ <- code.util.Helper.booleanToFuture( + s"$InvalidUserId user_id names a consent user (an agent identity minted by a Consent). Accounts are held by humans - use the granting user's USER_ID.", + failCode = 400, cc = Some(cc))(!appUser.exists(_.isConsentUser)) customer <- unboxOptionOBPReturnType(postedData.customer_id.map(NewStyle.function.getCustomerByCustomerId(_, Some(cc)))) (accountApplication, _) <- NewStyle.function.createAccountApplication( productCode = ProductCode(postedData.product_code), @@ -4301,10 +4321,16 @@ object Http4s310 { (accountBox, _) <- Connector.connector.vend.checkBankAccountExists(bank.bankId, AccountId(accountIdStr), Some(cc)) _ <- code.util.Helper.booleanToFuture(AccountIdAlreadyExists, cc = Some(cc)) { accountBox.isEmpty } loggedInUserId = user.userId - userIdAccountOwner = if (body.user_id.nonEmpty) body.user_id else loggedInUserId + // Implicit owner resolves to the HUMAN: under a Consent the caller is the + // per-consent shadow, and an account held by it strands when the consent dies. + userIdAccountOwner = if (body.user_id.nonEmpty) body.user_id else cc.accountableUserId _ <- code.util.Helper.booleanToFuture(InvalidAccountIdFormat, cc = Some(cc)) { isValidID(accountIdStr) } _ <- code.util.Helper.booleanToFuture(InvalidBankIdFormat, cc = Some(cc)) { isValidID(bankIdStr) } (accountOwner, _) <- NewStyle.function.findByUserId(userIdAccountOwner, Some(cc)) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + _ <- code.util.Helper.booleanToFuture( + s"$InvalidUserId user_id names a consent user (an agent identity minted by a Consent). Accounts are held by humans - use the granting user's USER_ID.", + failCode = 400, cc = Some(cc))(!accountOwner.isConsentUser) _ <- if (userIdAccountOwner == loggedInUserId) Future.successful(Full(())) else code.util.Helper.booleanToFuture( s"$UserHasMissingRoles $canCreateAccount or create account for self", @@ -5118,7 +5144,7 @@ object Http4s310 { .orElse(getObpConnectorLoopback.run(req)) } - val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allOwnRoutes) + val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allOwnRoutes)) // ─── path-rewriting bridge: /obp/v3.1.0/… → /obp/v3.0.0/… ────────────── diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala index d821649e0d..b54ddd7662 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala @@ -35,6 +35,7 @@ import code.api.v1_4_0.JSONFactory1_4_0 import code.DynamicEndpoint.DynamicEndpointSwagger import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.{APIUtil, CallContext, CustomJsonFormats, NewStyle} import code.api.v4_0_0.JSONFactory400._ import code.DynamicData.DynamicData @@ -3018,6 +3019,33 @@ object Http4s400 { // first synchronous read of `SS.user` captures the cc.user, then the Future chain // runs normally on any thread. + // Resolves the view createTransactionRequest needs, unit-testable without a live Mapper + // connection: `lookup` is production's real `Views.views.vend.systemView(...).or(...)` call + // in the route below, and a stub in the test. + // + // `lookup()` runs OUTSIDE tryons's blanket exception catch, not inside it. tryons/tryo catch + // any Exception the wrapped block raises and report it via the given failCode regardless of + // cause -- wrapping the DB call itself made a connection-pool exhaustion, a transient SQL + // error, or a Mapper bug indistinguishable from a genuine "no such view" and reported ALL of + // them as 404. `Future(lookup())` still catches an exception from `lookup()` (standard + // Future-block semantics), but as an ordinary failed Future carrying the ORIGINAL exception, + // untouched -- so it falls through to ErrorResponseConverter's catch-all (500), the same as + // any other unexpected server-side failure. Only a lookup that SUCCEEDS and returns an empty + // Box is a genuine client-side "not found", and only that case is explicitly mapped to 404 + // via tryons below (whose wrapped block cannot itself throw for any other reason -- it only + // ever raises the NoSuchElementException it constructs). + private[v4_0_0] def resolveCreateTransactionRequestView( + viewIdStr: String, + lookup: () => Box[View] + )(implicit cc: CallContext): Future[View] = + Future(lookup()).flatMap { + case Full(v) => Future.successful(v) + case _ => + NewStyle.function.tryons(s"$ViewNotFound Current view_id($viewIdStr)", 404, Some(cc)) { + throw new NoSuchElementException(s"view_id($viewIdStr)") + } + } + lazy val createTransactionRequest: HttpRoutes[IO] = HttpRoutes.of[IO] { // GRANT_VIEW_ID in the ResourceDoc URL → middleware skips view validation. // Lift's v4 endpoint does no view-access check upfront; it lets @@ -3041,24 +3069,29 @@ object Http4s400 { EndpointHelpers.executeFutureCreated(req) { val bodyStr = cc.httpBody.getOrElse("") for { - user <- Future { cc.user.openOrThrowException(AuthenticatedUserIsRequired) } - bank <- Future { cc.bank.getOrElse(throw new RuntimeException(BankNotFound)) } - account <- Future { cc.bankAccount.getOrElse(throw new RuntimeException(BankAccountNotFound)) } + // These four used to throw raw exceptions, which the converter can only render as + // OBP-50000 / HTTP 500. Every one of them is a client-side condition -- not + // authenticated, no such bank, no such account, no such view -- and a 500 tells a + // caller with retry logic to keep sending a request that cannot succeed. This is a + // payment path, so that retry loop is the expensive kind. + user <- NewStyle.function.tryons(AuthenticatedUserIsRequired, 401, Some(cc)) { + cc.user.openOrThrowException(AuthenticatedUserIsRequired) + } + bank <- NewStyle.function.tryons(BankNotFound, 404, Some(cc)) { + cc.bank.getOrElse(throw new NoSuchElementException(bankIdStr)) + } + account <- NewStyle.function.tryons(BankAccountNotFound, 404, Some(cc)) { + cc.bankAccount.getOrElse(throw new NoSuchElementException(accountIdStr)) + } json <- NewStyle.function.tryons( s"$InvalidJsonFormat Empty or invalid request body.", 400, Some(cc)) { com.openbankproject.commons.util.JsonAliases.parse(bodyStr) } transactionRequestType = TransactionRequestType(transactionRequestTypeStr) - view <- Future { - // System views (owner, accountant, etc.) and custom views (e.g. VRP - // `_vrp-…` views) are stored separately. Try system first; fall back - // to the account-scoped custom view. SS.init only needs *some* View - // instance — the connector reads viewId from the parameter, not the - // View object — so a soft fallback is fine here. + view <- resolveCreateTransactionRequestView(viewIdStr, () => Views.views.vend.systemView(ViewId(viewIdStr)) .or(Views.views.vend.customView(ViewId(viewIdStr), BankIdAccountId(account.bankId, account.accountId))) - .openOrThrowException(s"$ViewNotFound Current view_id($viewIdStr)") - } + ) // SS.init populates Lift thread-globals (used by `SS.user` inside the // connector). The connector's first line `SS.user` resolves synchronously // inside this block, capturing the user; subsequent flatMap stages run on @@ -6392,7 +6425,14 @@ object Http4s400 { case req @ GET -> `prefixPath` / "banks" / _ / "user-invitations" / secretLink => EndpointHelpers.withUserAndBank(req) { (_, bank, cc) => for { - (invitation, _) <- NewStyle.function.getUserInvitation(bank.bankId, secretLink.toLong, Some(cc)) + // `secretLink.toLong` used to run unguarded, so any non-numeric path segment left a + // NumberFormatException to escape as OBP-50000 / HTTP 500 -- a malformed identifier + // reported to the caller as a server fault, which tells a client with retry logic to + // keep sending a request that can never succeed. + secret <- NewStyle.function.tryons(s"$InvalidNumber Invalid SECRET_LINK: it must be a number.", 400, Some(cc)) { + secretLink.toLong + } + (invitation, _) <- NewStyle.function.getUserInvitation(bank.bankId, secret, Some(cc)) } yield JSONFactory400.createUserInvitationJson(invitation) } } @@ -7095,7 +7135,7 @@ object Http4s400 { "Get My Api Collection Endpoint", s"""Get Api Collection Endpoint By API_COLLECTION_NAME and OPERATION_ID. | - |${userAuthenticationMessage(false)} + |${userAuthenticationMessage(true)} |""".stripMargin, EmptyBody, apiCollectionEndpointJson400, @@ -7113,7 +7153,7 @@ object Http4s400 { "Get Api Collection Endpoints", s"""Get Api Collection Endpoints By API_COLLECTION_ID. | - |${userAuthenticationMessage(false)} + |${userAuthenticationMessage(true)} |""".stripMargin, EmptyBody, apiCollectionEndpointsJson400, @@ -9374,12 +9414,19 @@ object Http4s400 { case _ => true } } + // Fail fast with a clean 400 before attempting compilation, rather than surfacing an + // unsupported programming_lang only as a generic DynamicCodeCompileFail. + _ <- code.util.Helper.booleanToFuture( + s"""$DynamicCodeLangNotSupport programming_lang ${body.programmingLang}, currently supported languages: Scala, Java""", + cc = Some(cc)) { + Set("", "scala", "Scala", "java", "Java").contains(body.programmingLang) + } } yield () } private def compileDynamicResourceDoc(body: JsonDynamicResourceDoc, cc: CallContext): Unit = { try { - CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody).validateDependency() + CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang).validateDependency() } catch { case e: JsonResponseException => throw e case e: Exception => @@ -10137,10 +10184,16 @@ object Http4s400 { com.openbankproject.commons.util.JsonAliases.parse(rawBody).extract[code.api.v3_1_0.CreateAccountRequestJsonV310] } loggedInUserId = cc.userId + // Implicit owner resolves to the HUMAN: under a Consent the caller is the + // per-consent shadow, and an account held by it strands when the consent dies. userIdAccountOwner = if (createAccountJson.user_id.nonEmpty) createAccountJson.user_id - else loggedInUserId + else cc.accountableUserId (postedOrLoggedInUser, callContext) <- NewStyle.function.findByUserId(userIdAccountOwner, Some(cc)) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + _ <- code.util.Helper.booleanToFuture( + s"$InvalidUserId user_id names a consent user (an agent identity minted by a Consent). Accounts are held by humans - use the granting user's USER_ID.", + failCode = 400, cc = Some(cc))(!postedOrLoggedInUser.isConsentUser) _ <- if (userIdAccountOwner == loggedInUserId) Future.successful(Full(())) else NewStyle.function.hasEntitlement( bankId.value, loggedInUserId, canCreateAccount, callContext, @@ -10207,10 +10260,16 @@ object Http4s400 { com.openbankproject.commons.util.JsonAliases.parse(rawBody).extract[SettlementAccountRequestJson] } loggedInUserId = cc.userId + // Implicit owner resolves to the HUMAN: under a Consent the caller is the + // per-consent shadow, and an account held by it strands when the consent dies. userIdAccountOwner = if (createAccountJson.user_id.nonEmpty) createAccountJson.user_id - else loggedInUserId + else cc.accountableUserId (postedOrLoggedInUser, callContext) <- NewStyle.function.findByUserId(userIdAccountOwner, Some(cc)) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + _ <- code.util.Helper.booleanToFuture( + s"$InvalidUserId user_id names a consent user (an agent identity minted by a Consent). Accounts are held by humans - use the granting user's USER_ID.", + failCode = 400, cc = Some(cc))(!postedOrLoggedInUser.isConsentUser) _ <- if (userIdAccountOwner == loggedInUserId) Future.successful(Full(())) else NewStyle.function.hasEntitlement(bankId.value, loggedInUserId, canCreateSettlementAccountAtOneBank, callContext) initialBalanceAsString = createAccountJson.balance.amount @@ -11097,7 +11156,7 @@ object Http4s400 { .orElse(createUserInvitation.run(req)) } - lazy val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(allOwnRoutes) + lazy val allRoutesWithMiddleware: HttpRoutes[IO] = ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allOwnRoutes)) // ─── nameOf-compatibility aliases ──────────────────────────────────────── // These vals have no Lift counterpart in Http4s400 but are referenced by diff --git a/obp-api/src/main/scala/code/api/v5_0_0/Http4s500.scala b/obp-api/src/main/scala/code/api/v5_0_0/Http4s500.scala index 1de94f93d0..b5fd379833 100644 --- a/obp-api/src/main/scala/code/api/v5_0_0/Http4s500.scala +++ b/obp-api/src/main/scala/code/api/v5_0_0/Http4s500.scala @@ -14,6 +14,7 @@ import code.api.util.ErrorMessages import code.api.util.ErrorMessages._ import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware +import code.api.util.http4s.IdempotencyMiddleware import code.api.util.newstyle.ViewNewStyle import code.api.util.{APIUtil, ConsentJWT, ConsentView, Consent, CustomJsonFormats, JwtUtil, NewStyle, OBPBankId, SecureRandomUtil} import code.api.v2_1_0.JSONFactory210 @@ -463,18 +464,21 @@ object Http4s500 { postJson.bank_routings.getOrElse(Nil).filterNot(_.scheme == "BIC").headOption.map(_.address).getOrElse(""), Some(cc) ) - entitlements <- NewStyle.function.getEntitlementsByUserId(cc.userId, Some(cc)) + // Creator grants target the HUMAN (see v6.0.0 createBank): under a Consent the + // authenticated user is a per-consent shadow, and roles granted to it are stranded. + humanUserId = cc.accountableUserId + entitlements <- NewStyle.function.getEntitlementsByUserId(humanUserId, Some(cc)) entitlementsByBank = entitlements.filter(_.bankId == postJson.id.getOrElse("")) _ <- entitlementsByBank.exists(_.roleName == CanCreateEntitlementAtOneBank.toString()) match { case true => Future.successful(()) case false => Future(Entitlement.entitlement.vend.addEntitlement( - postJson.id.getOrElse(""), cc.userId, CanCreateEntitlementAtOneBank.toString(), + postJson.id.getOrElse(""), humanUserId, CanCreateEntitlementAtOneBank.toString(), grantedByUserId = Some(cc.userId))) } _ <- entitlementsByBank.exists(_.roleName == CanReadDynamicResourceDocsAtOneBank.toString()) match { case true => Future.successful(()) case false => Future(Entitlement.entitlement.vend.addEntitlement( - postJson.id.getOrElse(""), cc.userId, CanReadDynamicResourceDocsAtOneBank.toString(), + postJson.id.getOrElse(""), humanUserId, CanReadDynamicResourceDocsAtOneBank.toString(), grantedByUserId = Some(cc.userId))) } } yield JSONFactory500.createBankJSON500(success) @@ -584,10 +588,16 @@ object Http4s500 { com.openbankproject.commons.util.JsonAliases.parse(cc.httpBody.getOrElse("")).extract[CreateAccountRequestJsonV500] } loggedInUserId = user.userId - userIdAccountOwner = createAccountJson.user_id.getOrElse(loggedInUserId) + // Implicit owner resolves to the HUMAN: under a Consent the caller is the + // per-consent shadow, and an account held by it strands when the consent dies. + userIdAccountOwner = createAccountJson.user_id.getOrElse(cc.accountableUserId) _ <- Helper.booleanToFuture(InvalidAccountIdFormat, cc = Some(cc)) { isValidID(accountId.value) } _ <- Helper.booleanToFuture(InvalidBankIdFormat, cc = Some(cc)) { isValidID(accountId.value) } (postedOrLoggedInUser, _) <- NewStyle.function.findByUserId(userIdAccountOwner, Some(cc)) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + _ <- Helper.booleanToFuture( + s"$InvalidUserId user_id names a consent user (an agent identity minted by a Consent). Accounts are held by humans - use the granting user's USER_ID.", + failCode = 400, cc = Some(cc))(!postedOrLoggedInUser.isConsentUser) _ <- if (userIdAccountOwner == loggedInUserId) Future.successful(Full(())) else Helper.booleanToFuture( s"${UserHasMissingRoles} $canCreateAccount", failCode = 403, cc = Some(cc)) { @@ -2353,7 +2363,7 @@ object Http4s500 { } val allRoutesWithMiddleware: HttpRoutes[IO] = - ResourceDocMiddleware.apply(resourceDocs)(allRoutes) + ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes)) // ─── path-rewriting bridge: /obp/v5.0.0/… → /obp/v4.0.0/… ───────────── // Cascades inherited (v1.2.1–v4.0.0) endpoints through the http4s versions diff --git a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala index 96e1d695d0..fb10602e37 100644 --- a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala +++ b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala @@ -13,7 +13,7 @@ import code.api.util.ApiTag._ import code.api.util.ErrorMessages import code.api.util.ErrorMessages._ import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} -import code.api.util.http4s.{ResourceDocMiddleware, ResourceDocMatcher} +import code.api.util.http4s.{IdempotencyMiddleware, ResourceDocMatcher, ResourceDocMiddleware} import code.api.util.newstyle.{BalanceNewStyle, RegulatedEntityAttributeNewStyle, ViewNewStyle} import code.api.util.newstyle.RegulatedEntityNewStyle.{createRegulatedEntityNewStyle, deleteRegulatedEntityNewStyle, getRegulatedEntitiesNewStyle, getRegulatedEntityByEntityIdNewStyle} import code.api.util.newstyle.Consumer.createConsumerNewStyle @@ -3041,6 +3041,39 @@ object Http4s510 { http4sPartialFunction = Some(createMyConsumer) ) + // Walks a Throwable's cause chain looking for a JVM/security-provider configuration problem + // (the requested algorithm or provider is unavailable) rather than anything about the + // caller-supplied certificate or JWT. `RSASSAVerifier`/`SignedJWT.verify` wrap + // NoSuchAlgorithmException in a JOSEException when the JVM's registered security providers + // don't have the requested signature algorithm (a hardened/FIPS JRE, a stripped provider + // list, a provider-registration bug) -- a server/environment fault that has nothing to do + // with whether this particular client's certificate is well-formed. + private[v5_1_0] def hasSecurityProviderCause(t: Throwable): Boolean = + Iterator.iterate(t)(_.getCause).takeWhile(_ != null).exists { + case _: java.security.NoSuchAlgorithmException => true + case _: java.security.NoSuchProviderException => true + case _ => false + } + + // `JwtUtil.verifyJwt` does not merely return false for a bad certificate -- it can THROW at + // several points (PEM parsing, JWT parsing, key extraction, signature verification), and a + // client-malformed certificate or JWT is exactly what most of those throws mean. But wrapping + // the whole call in tryons(..., 400, ...) also converted a JVM/security-provider failure (see + // hasSecurityProviderCause) into the same 400 -- telling a caller their input was bad when + // the truth is the server's environment cannot perform this verification for ANY caller. + // `verify` is a thunk rather than a direct call so this is testable without live PEM/JWT + // material: production passes `() => JwtUtil.verifyJwt(jwt, pem)`, the test a stub that + // throws a chosen exception. + private[v5_1_0] def resolveJwtSignatureValid( + verify: () => Boolean + )(implicit cc: code.api.util.CallContext): Future[Boolean] = + Future(verify()).recoverWith { + case t if hasSecurityProviderCause(t) => + Future.failed(t) + case t => + NewStyle.function.tryons(PostJsonIsNotSigned, 400, Some(cc)) { throw t } + } + val createConsumerDynamicRegistration: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ POST -> `prefixPath` / "dynamic-registration" / "consumers" => EndpointHelpers.executeFutureCreated(req) { @@ -3050,9 +3083,14 @@ object Http4s510 { com.openbankproject.commons.util.JsonAliases.parse(cc.httpBody.getOrElse("")).extract[ConsumerJwtPostJsonV510] } pem = APIUtil.`getPSD2-CERT`(cc.requestHeaders) - _ <- Helper.booleanToFuture(PostJsonIsNotSigned, 400, Some(cc)) { - JwtUtil.verifyJwt(postedJwt.jwt, pem.getOrElse("")) - } + // `verifyJwt` does not merely return false for a bad certificate -- it THROWS + // ("No PEM-encoded keys found") when the PSD2-CERT header is absent or unparseable, + // and booleanToFuture only guards the false case, so the exception escaped as + // OBP-50000 / HTTP 500. A missing or malformed client certificate is a client error; + // reporting it as a server fault tells a caller with retry logic to keep sending a + // request that cannot ever succeed. + signatureValid <- resolveJwtSignatureValid(() => JwtUtil.verifyJwt(postedJwt.jwt, pem.getOrElse(""))) + _ <- Helper.booleanToFuture(PostJsonIsNotSigned, 400, Some(cc)) { signatureValid } postedJson <- NewStyle.function.tryons(InvalidJsonFormat, 400, Some(cc)) { com.openbankproject.commons.util.JsonAliases.parse(JwtUtil.getSignedPayloadAsJson(postedJwt.jwt).getOrElse("{}")).extract[ConsumerPostJsonV510] } @@ -3195,6 +3233,12 @@ object Http4s510 { APIUtil.canGrantAccessToView(com.openbankproject.commons.model.BankIdAccountIdViewId(bankId, accountId, viewId), targetViewId, user, Some(cc)) } (targetUser, _) <- NewStyle.function.findByUserId(postJson.user_id, Some(cc)) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + // A consent user's account access comes ONLY from its Consent (materialised and + // revoked with it); access granted here would outlive nothing and confuse audits. + _ <- Helper.booleanToFuture( + s"$InvalidUserId user_id names a consent user (an agent identity minted by a Consent). Account access targets humans - a consent user's access comes only from its Consent.", + failCode = 400, cc = Some(cc))(!targetUser.isConsentUser) view <- if (isValidSystemViewId(targetViewId.value)) ViewNewStyle.systemView(targetViewId, Some(cc)) else ViewNewStyle.customView(targetViewId, BankIdAccountId(bankId, accountId), Some(cc)) addedView <- JSONFactory400.grantAccountAccessToUser(bankId, accountId, targetUser, view, Some(cc)) @@ -5326,7 +5370,7 @@ object Http4s510 { } val allRoutesWithMiddleware: HttpRoutes[IO] = - ResourceDocMiddleware.apply(resourceDocs)(allRoutes) + ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes)) // ─── path-rewriting bridge: /obp/v5.1.0/… → /obp/v5.0.0/… ───────────── lazy val v510ToV500Bridge: HttpRoutes[IO] = Kleisli[HttpF, Request[IO], Response[IO]] { req => diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index ff0f80087e..d3a2d4dc6d 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -32,7 +32,7 @@ import code.api.util.{APIUtil, CallContext, CustomJsonFormats, NewStyle} import code.api.util.ApiRole._ import code.api.util.ApiTag._ import code.api.util.ErrorMessages._ -import code.api.util.http4s.{ErrorResponseConverter, RequestScopeConnection, ResourceDocMiddleware, ResourceDocMatcher} +import code.api.util.http4s.{ErrorResponseConverter, IdempotencyMiddleware, RequestScopeConnection, ResourceDocMatcher, ResourceDocMiddleware} import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.newstyle.ViewNewStyle import code.api.v2_0_0.JSONFactory200 @@ -199,7 +199,6 @@ object Http4s600 { else "oidc_operator_user_ids" def entitlementRequestId: Option[String] = None def groupId: Option[String] = None - def process: Option[String] = None def grantedByUserId: Option[String] = None } } @@ -491,8 +490,10 @@ object Http4s600 { else Nil ) } yield { + // Creator grants target the HUMAN (see createBank): a per-consent shadow principal + // must not end up owning the entity's admin roles. crudRoles.foreach(role => - Entitlement.entitlement.vend.addEntitlement(dynamicEntity.bankId.getOrElse(""), cc.userId, role.toString(), + Entitlement.entitlement.vend.addEntitlement(dynamicEntity.bankId.getOrElse(""), cc.accountableUserId, role.toString(), grantedByUserId = Some(cc.userId))) JSONFactory600.createMyDynamicEntitiesJson(List(result: DynamicEntityCommons)).dynamic_entities.head } @@ -869,10 +870,15 @@ object Http4s600 { postJson.bank_routings.getOrElse(Nil).filterNot(_.scheme == "BIC").headOption.map(_.address).getOrElse(""), Some(cc) ) - entitlements <- NewStyle.function.getEntitlementsByUserId(cc.userId, Some(cc)) + // Creator grant goes to the HUMAN, not the authenticated principal: under a + // Consent the principal is a per-consent shadow user, and a role granted to it + // is stranded when the consent dies (and invisible to the human's next consent). + // grantedByUserId stays the principal — the audit trail records who acted. + humanUserId = cc.accountableUserId + entitlements <- NewStyle.function.getEntitlementsByUserId(humanUserId, Some(cc)) entitlementsByBank = entitlements.filter(_.bankId == postJson.bank_id) _ = if (!entitlementsByBank.exists(_.roleName == CanCreateEntitlementAtOneBank.toString)) - Entitlement.entitlement.vend.addEntitlement(postJson.bank_id, cc.userId, CanCreateEntitlementAtOneBank.toString, + Entitlement.entitlement.vend.addEntitlement(postJson.bank_id, humanUserId, CanCreateEntitlementAtOneBank.toString, grantedByUserId = Some(cc.userId)) } yield JSONFactory600.createBankJSON600(success) } @@ -1035,6 +1041,32 @@ object Http4s600 { } + // Resolves the portal URL used to build the reset-password link. Unit-testable without + // touching Props: `portalUrlBox` is production's real `APIUtil.getPortalUrl` call in the + // route below, and a fixed Box in the test. + // + // 503, not 400. A missing public_obp_portal_url/portal_external_url is an operator's + // configuration mistake, not this caller's -- the exact condition Http4s700's createTestEmail + // reports as 503 ("the server is not broken -- it is not configured to do this, and [a wrong + // code] tells a caller with retry logic that the fault is transient"). A bare + // Future.failed(new Exception(s"$IncompleteServerConfiguration ...")) resolves to 400: the + // message starts with "OBP-10056: ", which ErrorResponseConverter's OBP-prefix path promotes + // only to {401,403,408,429} and defaults everything else to 400 -- so the admin resetting a + // password is told their request was bad. tryons with an explicit failCode bypasses that + // default entirely. + private[v6_0_0] def resolveResetPasswordPortalUrl( + portalUrlBox: net.liftweb.common.Box[String] + )(implicit cc: CallContext): Future[String] = + portalUrlBox match { + case Full(url) => Future.successful(url) + case _ => + NewStyle.function.tryons( + s"$IncompleteServerConfiguration public_obp_portal_url (or legacy portal_external_url) is not set", + 503, Some(cc)) { + throw new NoSuchElementException("public_obp_portal_url") + } + } + // Route: POST /obp/v6.0.0/management/user/reset-password-url (201) lazy val resetPasswordUrl: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ POST -> `prefixPath` / "management" / "user" / "reset-password-url" => @@ -1061,10 +1093,7 @@ object Http4s600 { case _ => throw new Exception("User not found, not validated, or email mismatch") } } - portalUrl <- APIUtil.getPortalUrl match { - case Full(url) => Future.successful(url) - case _ => Future.failed(new Exception(s"$IncompleteServerConfiguration public_obp_portal_url (or legacy portal_external_url) is not set")) - } + portalUrl <- resolveResetPasswordPortalUrl(APIUtil.getPortalUrl) resetLink <- Future { val user: AuthUser = authUser user.uniqueId.set(java.util.UUID.randomUUID().toString.replace("-", "")) @@ -1943,7 +1972,14 @@ object Http4s600 { postJson <- NewStyle.function.tryons(s"$InvalidJsonFormat The Json body should be the PostGroupMembershipJsonV600", 400, Some(cc)) { com.openbankproject.commons.util.JsonAliases.parse(rawBody).extract[JSONFactory600.PostGroupMembershipJsonV600] } - _ <- NewStyle.function.findByUserId(userIdStr, Some(cc)) + (targetUser, _) <- NewStyle.function.findByUserId(userIdStr, Some(cc)) + // Group membership is for humans. A consent user (an agent identity minted by a + // Consent) cannot hold durable roles — addEntitlement would redirect the grant to + // its granting human anyway, and removal via the consent user's id would then find + // nothing. Reject explicitly so the caller targets the human on purpose. + _ <- Helper.booleanToFuture( + s"$InvalidUserId USER_ID names a consent user (an agent identity minted by a Consent). Group membership targets humans - use the granting user's USER_ID.", + 400, Some(cc))(!targetUser.isConsentUser) group <- Future(code.group.GroupTrait.group.vend.getGroup(postJson.group_id)) .map(unboxFullOrFail(_, Some(cc), s"$UnknownError Group not found", 404)) _ <- groupRoleCheck(group.bankId, user.userId, canAddUserToGroupAtOneBank, canAddUserToGroupAtAllBanks, cc) @@ -1955,9 +1991,12 @@ object Http4s600 { ent.roleName == roleName && ent.bankId == group.bankId.getOrElse("") }) if (!alreadyHas) { + // createdByProcess carries the provenance (was left at "manual", making + // group-born rows read as hand-granted before the duplicate `process` + // column was retired). Entitlement.entitlement.vend.addEntitlement( - group.bankId.getOrElse(""), userIdStr, roleName, "manual", - Some(user.userId), Some(postJson.group_id), Some("GROUP_MEMBERSHIP")) + group.bankId.getOrElse(""), userIdStr, roleName, Constant.group_membership, + Some(user.userId), Some(postJson.group_id)) (roleName, true) } else (roleName, false) } @@ -1983,8 +2022,10 @@ object Http4s600 { .map(unboxFullOrFail(_, Some(cc), s"$UnknownError Group not found", 404)) _ <- groupRoleCheck(group.bankId, user.userId, canRemoveUserFromGroupAtOneBank, canRemoveUserFromGroupAtAllBanks, cc) entitlements <- Future(Entitlement.entitlement.vend.getEntitlementsByUserId(userIdStr)) + // group_id alone identifies group-born rows (only group grants set it) and holds + // for legacy rows too; the old `process == GROUP_MEMBERSHIP` conjunct was redundant. groupEntitlements = entitlements.toOption.getOrElse(List.empty).filter(e => - e.groupId == Some(groupId) && e.process == Some("GROUP_MEMBERSHIP")) + e.groupId == Some(groupId)) _ <- Future.sequence(groupEntitlements.map(e => Future(Entitlement.entitlement.vend.deleteEntitlement(Full(e))))) } yield "" @@ -2485,7 +2526,13 @@ object Http4s600 { _ <- Helper.booleanToFuture(BusinessJustificationRequired, cc = Some(cc)) { postJson.business_justification.trim.nonEmpty } - (_, _) <- NewStyle.function.findByUserId(postJson.target_user_id, Some(cc)) + (targetUser, _) <- NewStyle.function.findByUserId(postJson.target_user_id, Some(cc)) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + // Reject at request creation so no approver ever sees a request that the grant + // step would refuse anyway. + _ <- Helper.booleanToFuture( + s"$InvalidUserId target_user_id names a consent user (an agent identity minted by a Consent). Account access targets humans - a consent user's access comes only from its Consent.", + failCode = 400, cc = Some(cc))(!targetUser.isConsentUser) _ <- Helper.booleanToFuture(AccountAccessRequestAlreadyExists, 409, Some(cc)) { code.accountaccessrequest.AccountAccessRequestTrait.accountAccessRequest.vend .getByUserAccountView(postJson.target_user_id, bankIdStr, accountIdStr, postJson.view_id) @@ -2535,6 +2582,11 @@ object Http4s600 { u.userId != request.requestorUserId } (targetUser, _) <- NewStyle.function.findByUserId(request.targetUserId, Some(cc)) + // Belt and braces with the creation-side guard: a request stored before that guard + // existed (or written another way) must still not be granted to a consent user. + _ <- Helper.booleanToFuture( + s"$InvalidUserId The request's target user is a consent user (an agent identity minted by a Consent). Account access targets humans - a consent user's access comes only from its Consent.", + failCode = 400, cc = Some(cc))(!targetUser.isConsentUser) // Win the INITIATED -> APPROVED transition BEFORE granting view access. The provider's // conditional UPDATE makes this request the single actioner; the loser of a concurrent // approve/reject race gets a 400 here with NO side effect. Granting first would leave @@ -2634,9 +2686,15 @@ object Http4s600 { code.api.cache.RedisMessaging.validateChannelName(channelName) } info <- Future(code.api.cache.RedisMessaging.channelInfo(channelName)) + // A plain RuntimeException here surfaced as OBP-50000 / HTTP 500. "The thing you asked + // for does not exist" is the textbook 404; a 500 says the server broke, and a client + // cannot tell from it that retrying is pointless. (count, ttl) <- info match { case Some((c, t)) => Future.successful((c, t)) - case None => Future.failed(new RuntimeException(s"Channel '$channelName' not found")) + case None => + NewStyle.function.tryons(s"$SignalChannelNotFound Channel '$channelName' not found.", 404, Some(cc)) { + throw new NoSuchElementException(channelName) + } } } yield SignalChannelInfoJsonV600(channelName, count, ttl) } @@ -4369,12 +4427,23 @@ object Http4s600 { case _ => true } } + // Mirrors Http4s400's validateDynamicResourceDocBody: fail fast on an unsupported + // programming_lang here too, rather than reporting `valid = true` for a language + // create would actually reject with 400 DynamicCodeLangNotSupport (CompiledObjects + // silently falls through to the Scala compile path for any value it doesn't + // recognise as Java, so an unsupported/misspelled language would otherwise still + // "validate" successfully as Scala). + _ <- Helper.booleanToFuture( + s"""$DynamicCodeLangNotSupport programming_lang ${body.programmingLang}, currently supported languages: Scala, Java""", + cc = Some(cc)) { + Set("", "scala", "Scala", "java", "Java").contains(body.programmingLang) + } } yield try { code.api.dynamic.endpoint.helper.CompiledObjects( - body.exampleRequestBody, body.successResponseBody, body.methodBody).validateDependency() + body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang).validateDependency() ValidateDynamicResourceDocSuccessJsonV600( valid = true, - message = "Dynamic Resource Doc method body is valid Scala and uses allowed dependencies.") + message = s"Dynamic Resource Doc method body is valid ${body.programmingLang} and uses allowed dependencies.") } catch { case e: code.api.JsonResponseException => val errorText = e.jsonResponse match { @@ -4453,7 +4522,8 @@ object Http4s600 { for { (_, _) <- NewStyle.function.findByUserId(userId, Some(cc)) entitlements <- Future(code.entitlement.Entitlement.entitlement.vend.getEntitlementsByUserId(userId)) - groupEntitlements = entitlements.toOption.getOrElse(List.empty).filter(_.process == Some("GROUP_MEMBERSHIP")) + // group_id alone identifies group-born rows (see removeUserFromGroup). + groupEntitlements = entitlements.toOption.getOrElse(List.empty).filter(_.groupId.isDefined) groupIds = groupEntitlements.flatMap(_.groupId).distinct _ <- Future.sequence { groupIds.flatMap { gid => @@ -5607,7 +5677,7 @@ object Http4s600 { entitlement_id = ent.entitlementId, role_name = ent.roleName, bank_id = ent.bankId, user_id = ent.userId, username = userBox.map(_.name).getOrElse(""), - group_id = ent.groupId, process = ent.process) + group_id = ent.groupId, created_by_process = ent.createdByProcess) } }) } yield GroupEntitlementsJsonV600(withUsernames) @@ -6321,7 +6391,7 @@ object Http4s600 { // Deferring index construction to first request (post object-init) lets every // registration land before the snapshot is taken. lazy val allRoutesWithMiddleware: HttpRoutes[IO] = - ResourceDocMiddleware.apply(resourceDocs)(allRoutes) + ResourceDocMiddleware.apply(resourceDocs)(IdempotencyMiddleware(allRoutes)) // ─── path-rewriting bridge: /obp/v6.0.0/… → /obp/v5.1.0/… ───────────── // Targets v5.1.0; Http4s510 has its own working cascade down to v5.0.0 → v4.0.0 → … @@ -8795,7 +8865,7 @@ object Http4s600 { | |9 user_id (if null ignore) | - |Authentication is Required. + |${userAuthenticationMessage(true)} | |""".stripMargin, EmptyBody, @@ -9358,7 +9428,6 @@ object Http4s600 { | |Only removes entitlements with: |- group_id matching GROUP_ID - |- process = "GROUP_MEMBERSHIP" | |Requires either: |- CanRemoveUserFromGroupAtAllBanks (for any group) @@ -9890,7 +9959,7 @@ object Http4s600 { | |Optional query parameter `tag` — filter to products that have the given tag (e.g. `?tag=featured`). Tag matching is case-insensitive. | - |${userAuthenticationMessage(!getApiProductsIsPublic)}""".stripMargin, + |${userAuthenticationMessage(true)}""".stripMargin, EmptyBody, apiProductsJsonV600, List(UnknownError), @@ -9908,7 +9977,7 @@ object Http4s600 { | |Optional query parameter `tag` — filter to products that carry the given tag (e.g. `?tag=featured`). Tag matching is case-insensitive. Repeat `tag=` to require multiple tags. | - |${userAuthenticationMessage(!getProductsIsPublic)}""".stripMargin, + |${userAuthenticationMessage(true)}""".stripMargin, EmptyBody, productsJsonV600, List(UnknownError), @@ -12527,7 +12596,7 @@ object Http4s600 { "Get User's Group Memberships", s"""Get all groups a user is a member of. | - |Returns groups where the user has entitlements with process = "GROUP_MEMBERSHIP". + |Returns groups where the user has entitlements carrying a group_id. | |The response includes: |- list_of_entitlements: entitlements the user currently has from this group membership @@ -13363,7 +13432,7 @@ object Http4s600 { |Properties with sensitive keys or values (containing ${APIUtil.sensitiveKeywords.mkString(", ")}) |are excluded from the response entirely. | - |Authentication is Required. + |${userAuthenticationMessage(true)} | |""".stripMargin, EmptyBody, @@ -13838,7 +13907,7 @@ object Http4s600 { user_id = "user-id-123", username = "susan.uk.29@example.com", group_id = Some("group-id-123"), - process = Some("GROUP_MEMBERSHIP") + created_by_process = "GROUP_MEMBERSHIP" ) ) ), diff --git a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala index a4961c34c0..0d126d57bf 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala @@ -472,6 +472,10 @@ case class MetricJsonV600( operation_id: String, api_instance_id: String, consent_reference_id: Option[String], + // Authentication scheme of the call: "Consent", "OAuth2", "OAuth1", "DirectLogin", + // "GatewayLogin", "DAuth", "Anonymous", "Other". Absent on rows written before the + // auth_type column existed. + auth_type: Option[String], // How the caller's certificate was established: "direct", "forwarded" or "none"; // absent when the request carried no certificate material. See PeerTrust.Resolution. certificate_trust: Option[String], @@ -1745,6 +1749,7 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { operation_id = operationId, api_instance_id = metric.getApiInstanceId(), consent_reference_id = Option(metric.getConsentReferenceId()).filter(_.nonEmpty), + auth_type = Option(metric.getAuthType()).filter(_.nonEmpty), certificate_trust = Option(metric.getCertificateTrust()).filter(_.nonEmpty), certificate_trust_detail = Option(metric.getCertificateTrustDetail()).filter(_.nonEmpty) ) @@ -2061,7 +2066,9 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { user_id: String, username: String, group_id: Option[String], - process: Option[String] + // The row's stored provenance, verbatim: "GROUP_MEMBERSHIP" for rows granted since + // provenance moved to created_by_process; legacy group rows show "manual". + created_by_process: String ) case class GroupEntitlementsJsonV600( diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala index 55f4c40f0b..c27c550700 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala @@ -287,12 +287,12 @@ object Http4s700 { // Response shapes reuse the v6 bank JSON (BankJson600 / BanksJsonV600). // ─── Delegation fan-down for /my/banks ─────────────────────────────────── - // Resolving UP (agent caller → the granting human) is cc.effectiveHumanUserId. + // Resolving UP (agent caller → the granting human) is cc.accountableUserId. // This is the fan DOWN: the human plus every agent user minted from any Consent the // human granted — i.e. all user ids whose creations belong to that human. Match the // result against CreatedByUserId. Reads only server-written columns // (MappedConsent.mUserId, ResourceUser.CreatedByConsentId); the input must be an - // already-resolved human id (cc.effectiveHumanUserId), never a raw caller value. + // already-resolved human id (cc.accountableUserId), never a raw caller value. private def humanAndAgentUserIds(humanUserId: String): List[String] = { val consentIds = Consents.consentProvider.vend.getConsentsByUser(humanUserId) @@ -336,7 +336,7 @@ object Http4s700 { // Quota binds to the human: banks created by the human directly or by any // of their consent-agents count toward the same limit — otherwise every // new consent would arrive with a fresh quota. - val creatorUserIds = humanAndAgentUserIds(cc.effectiveHumanUserId) + val creatorUserIds = humanAndAgentUserIds(cc.accountableUserId) MappedBank.count(ByList(MappedBank.CreatedByUserId, creatorUserIds)) } _ <- Helper.booleanToFuture(SelfServiceBankLimitReached, failCode = 403, cc = Some(cc)) { @@ -356,8 +356,11 @@ object Http4s700 { "", "", "", "", "", "", Some(cc) ) + // Creator grant targets the HUMAN (see v6.0.0 createBank): under a Consent the + // authenticated user is a per-consent shadow, and roles granted to it are stranded. _ <- Future(Entitlement.entitlement.vend.addEntitlement( - generatedName.bankId, cc.userId, canCreateEntitlementAtOneBank.toString())) + generatedName.bankId, cc.accountableUserId, canCreateEntitlementAtOneBank.toString(), + grantedByUserId = Some(cc.userId))) } yield JSONFactory600.createBankJSON600(bank) } } @@ -414,7 +417,7 @@ object Http4s700 { EndpointHelpers.withUser(req) { (user, cc) => for { banksCreatedByUser <- Future { - val creatorUserIds = humanAndAgentUserIds(cc.effectiveHumanUserId) + val creatorUserIds = humanAndAgentUserIds(cc.accountableUserId) MappedBank.findAll(ByList(MappedBank.CreatedByUserId, creatorUserIds)) } } yield JSONFactory600.createBanksJsonV600(banksCreatedByUser) @@ -484,7 +487,13 @@ object Http4s700 { case req @ POST -> `prefixPath` / "users" / userId / "entitlements" => EndpointHelpers.withUserAndBodyCreated[CreateEntitlementJSON, AnyRef](req) { (user, body, cc) => for { - (_, _) <- NewStyle.function.findByUserId(userId, Some(cc)) + (targetUser, _) <- NewStyle.function.findByUserId(userId, Some(cc)) + // Explicit target: fail loud rather than redirect. A consent user (an agent + // identity minted by a Consent) cannot hold durable roles — grant to the + // granting human instead. + _ <- Helper.booleanToFuture( + s"$InvalidUserId USER_ID names a consent user (an agent identity minted by a Consent). Entitlements target humans - use the granting user's USER_ID.", + failCode = 400, cc = Some(cc))(!targetUser.isConsentUser) role <- NewStyle.function.tryons( s"$InvalidJsonFormat Unknown role: ${body.role_name}. Possible roles: ${ApiRole.availableRoles.sorted.mkString(", ")}", 400, Some(cc)) { ApiRole.valueOf(body.role_name) } @@ -933,7 +942,6 @@ object Http4s700 { else "oidc_operator_user_ids" def entitlementRequestId: Option[String] = None def groupId: Option[String] = None - def process: Option[String] = None def grantedByUserId: Option[String] = None } } @@ -1140,7 +1148,7 @@ object Http4s700 { // human, then fan down — both via server-written columns only. (metrics, _) <- APIMetrics.getMetricsFromHttpParams( httpParams, cc.callContext, - lockedUserIds = Some(humanAndAgentUserIds(cc.effectiveHumanUserId))) + lockedUserIds = Some(humanAndAgentUserIds(cc.accountableUserId))) } yield JSONFactory600.createMetricsJsonV600(metrics) } } @@ -2417,14 +2425,17 @@ object Http4s700 { _ <- Helper.booleanToFuture(UserEmailAddressMissing, 400, Some(cc)) { toAddress.nonEmpty } + // 503, not 500. The server is not broken -- it is not configured to do this, and a + // 500 tells a caller with retry logic that the fault is transient. Neither of these + // resolves without an operator editing props. _ <- Helper.booleanToFuture( s"$IncompleteServerConfiguration portal_external_url is not set — signup-validation and password-reset emails will not be delivered.", - 500, Some(cc)) { + 503, Some(cc)) { portalUrlBox.isDefined } _ <- Helper.booleanToFuture( s"$IncompleteServerConfiguration mail.users.userinfo.sender.address is still the default 'noreply@example.com' — most SMTP servers will reject this From address.", - 500, Some(cc)) { + 503, Some(cc)) { fromAddress != "noreply@example.com" } sendOutcome <- Future { @@ -4197,8 +4208,14 @@ object Http4s700 { case None => Future.successful(AccountId(APIUtil.generateUUID())) } // CanCreateAccount is enforced by ResourceDocMiddleware from the doc. - ownerId = body.user_id.filter(_.trim.nonEmpty).getOrElse(user.userId) + // The implicit owner is the HUMAN: under a Consent the caller (user.userId) is the + // per-consent shadow, and an account held by it strands when the consent dies. + ownerId = body.user_id.filter(_.trim.nonEmpty).getOrElse(cc.accountableUserId) (owner, _) <- NewStyle.function.findByUserId(ownerId, Some(cc)) + // Explicit target: fail loud rather than redirect (see the entitlement endpoints). + _ <- Helper.booleanToFuture( + s"$InvalidUserId user_id names a consent user (an agent identity minted by a Consent). Accounts are held by humans - use the granting user's USER_ID.", + failCode = 400, cc = Some(cc))(!owner.isConsentUser) initialBalance <- NewStyle.function.tryons(InvalidAccountInitialBalance, 400, Some(cc)) { BigDecimal(body.balance.amount) } diff --git a/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnectorInternal.scala b/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnectorInternal.scala index 9d05ca346c..fc5febee1e 100644 --- a/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnectorInternal.scala +++ b/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnectorInternal.scala @@ -32,7 +32,7 @@ import com.openbankproject.commons.model._ import com.openbankproject.commons.model.enums.ChallengeType.OBP_TRANSACTION_REQUEST_CHALLENGE import com.openbankproject.commons.model.enums.TransactionRequestTypes._ import com.openbankproject.commons.model.enums.{TransactionRequestStatus, _} -import com.tesobe.CacheKeyFromArguments +import com.tesobe.{CacheKeyFromArguments, CacheKeyOmit} import net.liftweb.common._ import org.json4s.JsonAST.JValue import org.json4s.native.Serialization.write @@ -478,7 +478,14 @@ object LocalMappedConnectorInternal extends MdcLoggable { Full(cardList) } - def getCurrentFxRateCached(bankId: BankId, fromCurrencyCode: String, toCurrencyCode: String, callContext: Option[CallContext]): Box[FXRate] = { + // @CacheKeyOmit on callContext: the rate depends on the bank and the currency pair only, but + // CacheKeyFromArguments renders every un-annotated parameter into the key, and CallContext + // carries per-request state (startTime, correlationId, url, verb, ipAddress, user). Keying on + // it made the key unique per request: the cache could never hit, and every call wrote a fresh + // Redis entry that lived out code.fx.exchangeRate.cache.ttl.seconds. The generated connectors + // have always annotated their callContext (see ConnectorBuilderUtil); this hand-written site + // simply never did. + def getCurrentFxRateCached(bankId: BankId, fromCurrencyCode: String, toCurrencyCode: String, @CacheKeyOmit callContext: Option[CallContext]): Box[FXRate] = { /** * Please note that "var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)" * is just a temporary value field with UUID values in order to prevent any ambiguity. @@ -1526,7 +1533,13 @@ object LocalMappedConnectorInternal extends MdcLoggable { accountRoutings = Nil, callContext = callContext ) - _ <- code.model.dataAccess.BankAccountCreation.setAccountHolderAndRefreshUserAccountAccess(bankId, newAccountId, cc.get.user.head, callContext) + // Holder is the HUMAN: under a Consent cc.user is the per-consent shadow, and a + // holding account held by it would strand when the consent dies. For non-consent + // callers accountableUserId is the caller, so this is a no-op for them. + holdingAccountHolder = cc.flatMap(c => + code.users.Users.users.vend.getUserByUserId(c.accountableUserId).toOption + ).getOrElse(cc.get.user.head) + _ <- code.model.dataAccess.BankAccountCreation.setAccountHolderAndRefreshUserAccountAccess(bankId, newAccountId, holdingAccountHolder, callContext) // create attribute on holding account to link to releaser account _ <- NewStyle.function.createOrUpdateAccountAttribute( bankId = bankId, diff --git a/obp-api/src/main/scala/code/bankconnectors/package.scala b/obp-api/src/main/scala/code/bankconnectors/package.scala index 7d835d1097..1211117536 100644 --- a/obp-api/src/main/scala/code/bankconnectors/package.scala +++ b/obp-api/src/main/scala/code/bankconnectors/package.scala @@ -21,7 +21,7 @@ import net.liftweb.util.ThreadGlobal import scala.concurrent.Future import scala.reflect.runtime.universe.{MethodSymbol, Type, typeOf} -import scala.util.{Success => TrySuccess, Failure => TryFailure} +import scala.util.{Try, Success => TrySuccess, Failure => TryFailure} import com.openbankproject.commons.util.{ApiVersion, ReflectUtils} import com.openbankproject.commons.util.ReflectUtils._ import com.openbankproject.commons.util.Functions.Implicits._ @@ -46,115 +46,119 @@ package object bankconnectors extends MdcLoggable { //this object is a empty Connector implementation, just for supply default args object StubConnector extends Connector - val intercept: InvocationHandler = new InvocationHandler { - override def invoke(proxy: AnyRef, method: Method, args: Array[AnyRef]): AnyRef = { - if (method.getReturnType.getName == "scala.concurrent.Future" && !canOpenFuture(method.getName)) { - throw new RuntimeException(ServiceIsTooBusy + s"Current Service(${method.getName})") - } else { - if (method.getName.contains("$default$") || ConnectorProxy.isInheritedMember(method)) { - // The empty Connector implements both: the $default$ accessors it inherits, and the - // members Connector itself does not declare. Routing the latter would look them up as - // connector calls - and NPE on the way, since args is null for a no-arg method. - val connectorMethodResult = method.invoke(StubConnector, args:_*) - if (connectorMethodResult.isInstanceOf[Future[_]] && canOpenFuture(method.getName)) { - FutureUtil.futureWithLimits(connectorMethodResult.asInstanceOf[Future[_]], method.getName) - } - connectorMethodResult - } else { - val methodName = method.getName - val argNameToValue: Array[(String, AnyRef)] = method.getParameters.map(_.getName).zip(args) - // TODO: getConnectorNameAndMethodRouting is also called inside invokeMethod. - // Consider refactoring invokeMethod to accept a pre-resolved connectorName to avoid the duplicate lookup. - val (_, connectorName) = getConnectorNameAndMethodRouting(methodName, argNameToValue) - - // Extract correlationId from CallContext before entering any Future callback, - // because Lift's S.containerSession is unavailable in async contexts. - val correlationId: String = args.collectFirst { - case Some(cc: CallContext) => cc.correlationId - case Full(cc: CallContext) => cc.correlationId - }.getOrElse(getCorrelationId()) // fallback to Lift session if no CallContext in args - - // Record outbound (before call) - ConnectorCountsRedis.incrementOutbound(connectorName, methodName) - val t0 = System.currentTimeMillis() - - val (connectorMethodResult, methodSymbol) = invokeMethod(method, args) - - // Track metrics for Future results - if (connectorMethodResult.isInstanceOf[Future[_]]) { - val future = connectorMethodResult.asInstanceOf[Future[Any]] - future.onComplete { result => - val duration = System.currentTimeMillis() - t0 - val isSuccess = result match { - case TrySuccess(value) => !isFailureBox(value) - case TryFailure(_) => false - } - - // Record inbound - ConnectorCountsRedis.incrementInbound(connectorName, methodName, isSuccess) + // Record the outcome of a connector call: counters, plus optional detailed metric/trace persistence. + def recordConnectorInboundMetrics(connectorName: String, methodName: String, correlationId: String, + duration: Long, isSuccess: Boolean, args: Array[AnyRef]): Unit = { + ConnectorCountsRedis.incrementInbound(connectorName, methodName, isSuccess) + if (getPropsAsBoolValue("write_connector_metrics", false)) { + val params = extractKeyParams(args) + Future { + ConnectorMetricsProvider.metrics.vend.saveConnectorMetric( + connectorName, methodName, correlationId, now, duration, params, isSuccess) + } + } + } - // Record detailed metric to DB - if (getPropsAsBoolValue("write_connector_metrics", false)) { - val params = extractKeyParams(args) - Future { - ConnectorMetricsProvider.metrics.vend.saveConnectorMetric( - connectorName, methodName, correlationId, now, duration, params, isSuccess) - } - } + def recordConnectorTrace(connectorName: String, methodName: String, method: Method, args: Array[AnyRef], + duration: Long, isSuccess: Boolean, result: Try[Any]): Unit = { + if (getPropsAsBoolValue("write_connector_trace", false)) { + val outbound = serializeOutboundArgs(method, args) + val inbound = serializeInboundResult(result) + val correlationId = getCorrelationId() + val (detailUserId, detailHttpVerb, detailApiUrl) = extractCallContextInfo(args) + val bankIdValue = extractBankIdFromArgs(args) + Future { + ConnectorTraceProvider.saveConnectorTrace( + correlationId, connectorName, methodName, bankIdValue, + outbound, inbound, now, duration, isSuccess, + detailUserId, detailHttpVerb, detailApiUrl) + } + } + } - // Record connector trace (outbound/inbound messages) - if (getPropsAsBoolValue("write_connector_trace", false)) { - val outbound = serializeOutboundArgs(method, args) - val inbound = serializeInboundResult(result) - val correlationId = getCorrelationId() - val (detailUserId, detailHttpVerb, detailApiUrl) = extractCallContextInfo(args) - val bankIdValue = extractBankIdFromArgs(args) - Future { - ConnectorTraceProvider.saveConnectorTrace( - correlationId, connectorName, methodName, bankIdValue, - outbound, inbound, now, duration, isSuccess, - detailUserId, detailHttpVerb, detailApiUrl) - } - } - } - } else { - // Non-future (legacy Box) result - track synchronously - val duration = System.currentTimeMillis() - t0 - val isSuccess = !isFailureBox(connectorMethodResult) - - ConnectorCountsRedis.incrementInbound(connectorName, methodName, isSuccess) - - if (getPropsAsBoolValue("write_connector_metrics", false)) { - val params = extractKeyParams(args) - Future { - ConnectorMetricsProvider.metrics.vend.saveConnectorMetric( - connectorName, methodName, correlationId, now, duration, params, isSuccess) - } - } + // The empty Connector implements both: the $default$ accessors it inherits, and the + // members Connector itself does not declare. Routing the latter would look them up as + // connector calls - and NPE on the way, since args is null for a no-arg method. + def delegateToStub(method: Method, args: Array[AnyRef]): AnyRef = { + val connectorMethodResult = method.invoke(StubConnector, args:_*) + if (connectorMethodResult.isInstanceOf[Future[_]] && canOpenFuture(method.getName)) { + FutureUtil.futureWithLimits(connectorMethodResult.asInstanceOf[Future[_]], method.getName) + } + connectorMethodResult + } - // Record connector trace (outbound/inbound messages) - if (getPropsAsBoolValue("write_connector_trace", false)) { - val outbound = serializeOutboundArgs(method, args) - val inbound = serializeInboundResult(TrySuccess(connectorMethodResult)) - val correlationId = getCorrelationId() - val (detailUserId, detailHttpVerb, detailApiUrl) = extractCallContextInfo(args) - val bankIdValue = extractBankIdFromArgs(args) - Future { - ConnectorTraceProvider.saveConnectorTrace( - correlationId, connectorName, methodName, bankIdValue, - outbound, inbound, now, duration, isSuccess, - detailUserId, detailHttpVerb, detailApiUrl) - } - } - } - - if (connectorMethodResult.isInstanceOf[Future[_]] && canOpenFuture(method.getName)) { - FutureUtil.futureWithLimits(connectorMethodResult.asInstanceOf[Future[_]], method.getName) - } - logger.debug(s"do required field validation for ${methodSymbol.typeSignature}") - val apiVersion = ApiVersionHolder.getApiVersion - validateRequiredFields(connectorMethodResult, methodSymbol.returnType, apiVersion) + def routeToConnector(method: Method, args: Array[AnyRef]): AnyRef = { + val methodName = method.getName + val argNameToValue: Array[(String, AnyRef)] = method.getParameters.map(_.getName).zip(args) + // TODO: getConnectorNameAndMethodRouting is also called inside invokeMethod. + // Consider refactoring invokeMethod to accept a pre-resolved connectorName to avoid the duplicate lookup. + val (_, connectorName) = getConnectorNameAndMethodRouting(methodName, argNameToValue) + + // Extract correlationId from CallContext before entering any Future callback, + // because Lift's S.containerSession is unavailable in async contexts. + val correlationId: String = args.collectFirst { + case Some(cc: CallContext) => cc.correlationId + case Full(cc: CallContext) => cc.correlationId + }.getOrElse(getCorrelationId()) // fallback to Lift session if no CallContext in args + + // Record outbound (before call) + ConnectorCountsRedis.incrementOutbound(connectorName, methodName) + val t0 = System.currentTimeMillis() + + val (connectorMethodResult, methodSymbol) = invokeMethod(method, args) + + // Track metrics for Future results + if (connectorMethodResult.isInstanceOf[Future[_]]) { + val future = connectorMethodResult.asInstanceOf[Future[Any]] + future.onComplete { result => + val duration = System.currentTimeMillis() - t0 + val isSuccess = result match { + case TrySuccess(value) => !isFailureBox(value) + case TryFailure(_) => false } + recordConnectorInboundMetrics(connectorName, methodName, correlationId, duration, isSuccess, args) + recordConnectorTrace(connectorName, methodName, method, args, duration, isSuccess, result) + } + } else { + // Non-future (legacy Box) result - track synchronously + val duration = System.currentTimeMillis() - t0 + val isSuccess = !isFailureBox(connectorMethodResult) + recordConnectorInboundMetrics(connectorName, methodName, correlationId, duration, isSuccess, args) + recordConnectorTrace(connectorName, methodName, method, args, duration, isSuccess, TrySuccess(connectorMethodResult)) + } + + if (connectorMethodResult.isInstanceOf[Future[_]] && canOpenFuture(method.getName)) { + FutureUtil.futureWithLimits(connectorMethodResult.asInstanceOf[Future[_]], method.getName) + } + logger.debug(s"do required field validation for ${methodSymbol.typeSignature}") + val apiVersion = ApiVersionHolder.getApiVersion + validateRequiredFields(connectorMethodResult, methodSymbol.returnType, apiVersion) + } + + val intercept: InvocationHandler = new InvocationHandler { + override def invoke(proxy: AnyRef, method: Method, rawArgs: Array[AnyRef]): AnyRef = { + // `java.lang.reflect.Proxy` passes null for a method that declares no parameters; cglib, + // which this replaced, passed a zero-length array. Everything downstream treats args as a + // collection -- `.zip(args)`, `args.collectFirst`, `extractKeyParams(args)` -- and every + // one of those throws on null. + // + // isInheritedMember covers the members Connector does not declare, but a NO-ARGUMENT + // method that Connector DOES declare slips past it and lands in routeToConnector. + // Measured on GET /obp/v6.0.0/system/connector-method-names, which reads + // `connector.callableMethods`: 200 on the 2.12/cglib build, 500 on this one, with + // `Cannot invoke "scala.collection.IterableOnce.knownSize()" because "that" is null` -- + // which is `zip` being handed the null. + // + // Normalising to an empty array restores exactly what cglib did, which is what a + // toolchain migration owes its callers. `method.invoke(target, args: _*)` is unaffected: + // it compiles to Java varargs and an empty array means the same as null there. + val args: Array[AnyRef] = if (rawArgs == null) Array.empty[AnyRef] else rawArgs + if (method.getReturnType.getName == "scala.concurrent.Future" && !canOpenFuture(method.getName)) { + throw new RuntimeException(ServiceIsTooBusy + s"Current Service(${method.getName})") + } else if (method.getName.contains("$default$") || ConnectorProxy.isInheritedMember(method)) { + delegateToStub(method, args) + } else { + routeToConnector(method, args) } } } diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala index a96c164350..d42ef429fd 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala @@ -25,6 +25,9 @@ class DynamicResourceDoc extends LongKeyedMapper[DynamicResourceDoc] with IdPK w object Tags extends MappedString(this, 255) object Roles extends MappedString(this, 255) object MethodBody extends MappedText(this) + // Source language of MethodBody: "Scala" (default) or "Java". Mirrors DynamicMessageDoc.Lang / + // ConnectorMethod.programmingLang — same field name/width convention, see DynamicEndpoints. + object Lang extends MappedString(this, 50) // Provenance: who created / last updated this runtime-compiled endpoint, and a SHA-256 of the // (decoded) method body so tampering / drift is detectable. Set server-side from the CallContext // user — never from the request body. createdAt / updatedAt come from the CreatedUpdated trait. @@ -50,7 +53,12 @@ object DynamicResourceDoc extends DynamicResourceDoc with LongKeyedMetaMapper[Dy successResponseBody = Option(dynamicResourceDoc.SuccessResponseBody.get).filter(StringUtils.isNotBlank).map(json.parse), errorResponseBodies = dynamicResourceDoc.ErrorResponseBodies.get, tags = dynamicResourceDoc.Tags.get, - roles = dynamicResourceDoc.Roles.get + roles = dynamicResourceDoc.Roles.get, + // Rows created before the Lang column existed have NULL there, not "Scala" -- a bare + // Lang.get would surface that as an empty/null programming_lang instead of falling back to + // JsonDynamicResourceDoc's own "Scala" default, since an explicit null argument bypasses a + // case class default (that only applies when the argument is omitted entirely). + programmingLang = Option(dynamicResourceDoc.Lang.get).filter(StringUtils.isNotBlank).getOrElse("Scala") ) } diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala index 60e94a243a..aca89c7f76 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala @@ -33,7 +33,12 @@ case class JsonDynamicResourceDoc( successResponseBody: Option[JValue], errorResponseBodies: String, tags: String, - roles: String + roles: String, + // Source language of methodBody: "Scala" (default) or "Java". Mirrors + // JsonConnectorMethod.programmingLang / JsonDynamicMessageDoc.programmingLang. Appended last + // (not inserted alphabetically) so existing named-arg call sites and JSON payloads that predate + // this field keep compiling/deserializing unchanged. + programmingLang: String = "Scala" ) extends JsonFieldReName { def decodedMethodBody: String = URLDecoder.decode(methodBody, "UTF-8") } diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala index 47be5d0442..7b8d25a700 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala @@ -82,6 +82,7 @@ object MappedDynamicResourceDocProvider extends DynamicResourceDocProvider { .Tags(entity.tags) .Roles(entity.roles) .MethodBody(entity.methodBody) + .Lang(entity.programmingLang) // provenance is set here from the authenticated user + computed hash, not from `entity` .CreatedByUserId(createdByUserId.getOrElse(null)) .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) @@ -107,6 +108,7 @@ object MappedDynamicResourceDocProvider extends DynamicResourceDocProvider { .Tags(entity.tags) .Roles(entity.roles) .MethodBody(entity.methodBody) + .Lang(entity.programmingLang) // CreatedByUserId is left untouched; record who last changed the code + refresh the hash .UpdatedByUserId(updatedByUserId.getOrElse(null)) .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) diff --git a/obp-api/src/main/scala/code/entitlement/Entilement.scala b/obp-api/src/main/scala/code/entitlement/Entilement.scala index 211d227f0a..9fbe78ae4d 100644 --- a/obp-api/src/main/scala/code/entitlement/Entilement.scala +++ b/obp-api/src/main/scala/code/entitlement/Entilement.scala @@ -44,8 +44,7 @@ trait EntitlementProvider { // createdByProcess carries the provenance. Authorization is the // calling endpoint's responsibility, not this method's. grantedByUserId: Option[String] = None, - groupId: Option[String] = None, - process: Option[String] = None + groupId: Option[String] = None ): Box[Entitlement] def deleteDynamicEntityEntitlement( entityName: String, @@ -62,7 +61,6 @@ trait Entitlement { def createdByProcess: String def entitlementRequestId: Option[String] def groupId: Option[String] - def process: Option[String] /** user_id of the granter, when the grant was made by a person (directly * or as a self-grant). None for system-process grants and virtual diff --git a/obp-api/src/main/scala/code/entitlement/MappedEntitlements.scala b/obp-api/src/main/scala/code/entitlement/MappedEntitlements.scala index da3c09b385..7d032cb521 100644 --- a/obp-api/src/main/scala/code/entitlement/MappedEntitlements.scala +++ b/obp-api/src/main/scala/code/entitlement/MappedEntitlements.scala @@ -6,6 +6,7 @@ import code.api.util.ApiRole.{ CanCreateEntitlementAtOneBank } import code.api.util.{ErrorMessages, NotificationUtil} +import code.util.Helper.MdcLoggable import code.util.{MappedUUID, UUIDString} import net.liftweb.common.{Box, Failure, Full} import net.liftweb.mapper._ @@ -15,7 +16,7 @@ import scala.concurrent.Future import com.openbankproject.commons.ExecutionContext.Implicits.global import net.liftweb.common -object MappedEntitlementsProvider extends EntitlementProvider { +object MappedEntitlementsProvider extends EntitlementProvider with MdcLoggable { override def getEntitlement( bankId: String, userId: String, @@ -163,32 +164,57 @@ object MappedEntitlementsProvider extends EntitlementProvider { roleName: String, createdByProcess: String = "manual", grantedByUserId: Option[String] = None, - groupId: Option[String] = None, - process: Option[String] = None + groupId: Option[String] = None ): Box[Entitlement] = { // grantedByUserId is audit metadata, stored as-is: authorization is the // calling endpoint's responsibility. (Until 2026-08-09 an unused // grantorUserId parameter gated on the grantor's granting roles here — // no caller ever passed it, and the check ignored super admins, whose // granting rights are virtual and have no rows to find.) + + // On-behalf-of guard: a consent user (the per-consent principal a Consent-JWT + // authenticates as; its ResourceUser row carries CreatedByConsentId) must not + // accumulate durable roles — they strand when the consent dies, invisible to the + // human's next consent (see the simon.bank creator-grant incident, 2026-08-31). + // Any grant targeting one is redirected to the consent's granting human. The one + // legitimate writer of consent-user rows is the consent engine copying the + // consent's own scope, which tags itself Constant.consent_user and is exempt. + val targetUserId = + if (createdByProcess == code.api.Constant.consent_user) userId + else { + val grantingHumanUserId = for { + resourceUser <- code.model.dataAccess.ResourceUser.find( + By(code.model.dataAccess.ResourceUser.userId_, userId)) + consentId <- Full(resourceUser.CreatedByConsentId.get) + .filter(id => id != null && id.nonEmpty) + consent <- code.consent.Consents.consentProvider.vend.getConsentByConsentId(consentId) + humanUserId <- Full(consent.userId).filter(id => id != null && id.nonEmpty) + } yield humanUserId + grantingHumanUserId match { + case Full(humanUserId) => + logger.warn(s"addEntitlement: target user $userId is a consent user; granting role '$roleName' (bankId '$bankId', createdByProcess '$createdByProcess') to its granting human $humanUserId instead") + humanUserId + case _ => userId + } + } + def addEntitlementToUser(): Box[MappedEntitlement] = { val entitlement = MappedEntitlement.create .mBankId(bankId) - .mUserId(userId) + .mUserId(targetUserId) .mRoleName(roleName) .mCreatedByProcess(createdByProcess) grantedByUserId.foreach(g => entitlement.mGrantedByUserId(g)) groupId.foreach(gid => entitlement.mGroupId(gid)) - process.foreach(p => entitlement.mProcess(p)) tryo(entitlement.saveMe()) match { case Full(saved) => - NotificationUtil.sendEmailRegardingAssignedRole(userId, saved) + NotificationUtil.sendEmailRegardingAssignedRole(targetUserId, saved) Full(saved) case Failure(_, _, _) => // UniqueIndex(mBankId, mUserId, mRoleName) violated by concurrent grant — return the committed row MappedEntitlement.find( By(MappedEntitlement.mBankId, bankId), - By(MappedEntitlement.mUserId, userId), + By(MappedEntitlement.mUserId, targetUserId), By(MappedEntitlement.mRoleName, roleName) ) case other => other @@ -217,10 +243,10 @@ class MappedEntitlement override def defaultValue = "" } - object mProcess extends MappedString(this, 255) { - override def dbColumnName = "process" - override def defaultValue = "" - } + // The legacy "process" DB column (a duplicate of createdByProcess written only by the + // Groups feature) is no longer mapped: group rows are identified by group_id, and + // provenance lives in createdByProcess. The column itself can be dropped from the DB + // whenever convenient. object entitlement_request_id extends MappedUUID(this) { override def dbColumnName = "entitlement_request_id" @@ -243,10 +269,6 @@ class MappedEntitlement val gid = mGroupId.get if (gid == null || gid.isEmpty) None else Some(gid) } - override def process: Option[String] = { - val p = mProcess.get - if (p == null || p.isEmpty) None else Some(p) - } override def grantedByUserId: Option[String] = { val g = mGrantedByUserId.get if (g == null || g.isEmpty) None else Some(g) diff --git a/obp-api/src/main/scala/code/metrics/APIMetrics.scala b/obp-api/src/main/scala/code/metrics/APIMetrics.scala index c8b26ae5e8..c3dfa8e784 100644 --- a/obp-api/src/main/scala/code/metrics/APIMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/APIMetrics.scala @@ -101,7 +101,8 @@ trait APIMetrics { apiInstanceId: String, consentReferenceId: String, certificateTrust: String, - certificateTrustDetail: String): Unit + certificateTrustDetail: String, + authType: String): Unit def saveMetricsArchive(primaryKey: Long, userId: String, @@ -123,7 +124,8 @@ trait APIMetrics { apiInstanceId: String, consentReferenceId: String, certificateTrust: String, - certificateTrustDetail: String + certificateTrustDetail: String, + authType: String ): Boolean // //TODO: ordering of list? should this be by date? currently not enforced @@ -183,6 +185,9 @@ trait APIMetric { def getConsentReferenceId(): String def getCertificateTrust(): String def getCertificateTrustDetail(): String + // Authentication scheme of the call — "Consent", "OAuth2", "OAuth1", "DirectLogin", + // "GatewayLogin", "DAuth", "Anonymous" or "Other". Scheme only, never credentials. + def getAuthType(): String } @@ -210,7 +215,7 @@ case class AggregateMetrics( minResponseTime: Double, maxResponseTime: Double, // Distinct humans behind the calls: consent-borne rows are attributed to the granting - // (on-behalf-of) user via the consent table, mirroring CallContext.effectiveHumanUserId. + // (on-behalf-of) user via the consent table, mirroring CallContext.accountableUserId. distinctUserCount: Int, distinctConsumerCount: Int, // Calls that arrived under a consent (metric.consent_reference_id not null), and how many diff --git a/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala b/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala index fb80d918d3..925cb6abed 100644 --- a/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala @@ -15,7 +15,8 @@ object ElasticsearchMetrics extends APIMetrics { override def saveMetric(userId: String, url: String, date: Date, duration: Long, userName: String, appName: String, developerEmail: String, consumerId: String, implementedByPartialFunction: String, implementedInVersion: String, verb: String, httpCode: Option[Int], correlationId: String, responseBody: String, sourceIp: String, targetIp: String, apiInstanceId: String, consentReferenceId: String, - certificateTrust: String, certificateTrustDetail: String): Unit = { + certificateTrust: String, certificateTrustDetail: String, + authType: String): Unit = { if (APIUtil.getPropsAsBoolValue("allow_elasticsearch", false) && APIUtil.getPropsAsBoolValue("allow_elasticsearch_metrics", false) ) { //TODO ,need to be fixed now add more parameters es.indexMetric(userId, url, date, duration, userName, appName, developerEmail, correlationId, apiInstanceId) @@ -28,7 +29,8 @@ object ElasticsearchMetrics extends APIMetrics { apiInstanceId: String, consentReferenceId: String, certificateTrust: String, - certificateTrustDetail: String): Boolean = ??? + certificateTrustDetail: String, + authType: String): Boolean = ??? // override def getAllGroupedByUserId(): Map[String, List[APIMetric]] = { // //TODO: replace the following with valid ES query diff --git a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala index e890e5e2e6..e6ad043619 100644 --- a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala @@ -114,7 +114,8 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ override def saveMetric(userId: String, url: String, date: Date, duration: Long, userName: String, appName: String, developerEmail: String, consumerId: String, implementedByPartialFunction: String, implementedInVersion: String, verb: String, httpCode: Option[Int], correlationId: String, responseBody: String, sourceIp: String, targetIp: String, apiInstanceId: String, consentReferenceId: String, - certificateTrust: String, certificateTrustDetail: String): Unit = { + certificateTrust: String, certificateTrustDetail: String, + authType: String): Unit = { // A correlation id is expected on every metric. Rows without one cannot be moved // to the archive later (its correlationId column requires a UUID), so flag it at // write time where the source of the missing id can actually be traced. @@ -142,7 +143,8 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ apiInstanceId = apiInstanceId, consentReferenceId = consentReferenceId, certificateTrust = certificateTrust, - certificateTrustDetail = certificateTrustDetail + certificateTrustDetail = certificateTrustDetail, + authType = authType ) ) } @@ -153,7 +155,8 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ verb: String, httpCode: Option[Int], correlationId: String, responseBody: String, sourceIp: String, targetIp: String, apiInstanceId: String, consentReferenceId: String, - certificateTrust: String, certificateTrustDetail: String): Boolean = { + certificateTrust: String, certificateTrustDetail: String, + authType: String): Boolean = { // Fix: dedup by the source metric's primary key stored in `metricId`, NOT by the // archive's own auto-increment `id`. The two are unrelated id-spaces; matching on // `id` overwrites an unrelated archived row once the archive's id sequence grows @@ -181,6 +184,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ .consentReferenceId(consentReferenceId) .certificateTrust(certificateTrust) .certificateTrustDetail(certificateTrustDetail) + .authType(authType) httpCode match { case Some(code) => metric.httpCode(code) @@ -431,7 +435,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ // The LEFT JOIN attributes consent-borne calls to the granting (on-behalf-of) human: // metric.userid records the AUTHENTICATED principal, which under a consent is the // consent's own shadow user. COALESCE(consent.muserid, metric.userid) resolves such rows - // to the granting human at read time, mirroring CallContext.effectiveHumanUserId. (Rows + // to the granting human at read time, mirroring CallContext.accountableUserId. (Rows // written 2026-08 only, while toLight briefly recorded the human, resolve identically.) // The consent side of the join is unique-indexed on consent_reference_id, so the join // cannot fan out rows. @@ -827,6 +831,13 @@ class MappedMetric extends APIMetric with LongKeyedMapper[MappedMetric] with IdP } // The specifics behind certificateTrust (PeerTrust.Resolution.detail): the forwarding proxy's // canonical subject DN for "forwarded", the rejection reason for "none". Null for "direct". + // Authentication scheme of the call (scheme only, never credentials): "Consent", + // "OAuth2", "OAuth1", "DirectLogin", "GatewayLogin", "DAuth", "Anonymous", "Other". + // Null on rows written before the column existed. + object authType extends MappedString(this, 32) { + override def dbColumnName = "auth_type" + override def defaultValue = null + } object certificateTrustDetail extends MappedString(this, 255) { override def dbColumnName = "certificate_trust_detail" override def defaultValue = null @@ -853,6 +864,7 @@ class MappedMetric extends APIMetric with LongKeyedMapper[MappedMetric] with IdP override def getConsentReferenceId(): String = consentReferenceId.get override def getCertificateTrust(): String = certificateTrust.get override def getCertificateTrustDetail(): String = certificateTrustDetail.get + override def getAuthType(): String = authType.get } object MappedMetric extends MappedMetric with LongKeyedMetaMapper[MappedMetric] { @@ -921,6 +933,13 @@ class MetricArchive extends APIMetric with LongKeyedMapper[MetricArchive] with I override def dbColumnName = "certificate_trust" override def defaultValue = null } + // Authentication scheme of the call (scheme only, never credentials): "Consent", + // "OAuth2", "OAuth1", "DirectLogin", "GatewayLogin", "DAuth", "Anonymous", "Other". + // Null on rows written before the column existed. + object authType extends MappedString(this, 32) { + override def dbColumnName = "auth_type" + override def defaultValue = null + } object certificateTrustDetail extends MappedString(this, 255) { override def dbColumnName = "certificate_trust_detail" override def defaultValue = null @@ -948,6 +967,7 @@ class MetricArchive extends APIMetric with LongKeyedMapper[MetricArchive] with I override def getConsentReferenceId(): String = consentReferenceId.get override def getCertificateTrust(): String = certificateTrust.get override def getCertificateTrustDetail(): String = certificateTrustDetail.get + override def getAuthType(): String = authType.get } object MetricArchive extends MetricArchive with LongKeyedMetaMapper[MetricArchive] { override def dbIndexes = diff --git a/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala b/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala index d1a05d5ba2..36107eafe1 100644 --- a/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala +++ b/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala @@ -43,7 +43,8 @@ object MetricBatchWriter extends MdcLoggable { apiInstanceId: String, consentReferenceId: String, certificateTrust: String, - certificateTrustDetail: String + certificateTrustDetail: String, + authType: String ) private val queue = new ConcurrentLinkedQueue[MetricRow]() @@ -106,8 +107,8 @@ object MetricBatchWriter extends MdcLoggable { developeremail, consumerid, implementedbypartialfunction, implementedinversion, verb, httpcode, correlationid, responsebody, sourceip, targetip, apiinstanceid, consent_reference_id, - certificate_trust, certificate_trust_detail - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + certificate_trust, certificate_trust_detail, auth_type + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """ // Use Option[String] so Doobie handles nullable fields via Put[Option[String]] @@ -117,7 +118,7 @@ object MetricBatchWriter extends MdcLoggable { Option[String], Option[String], Option[String], Option[String], Option[String], Int, Option[String], Option[String], Option[String], Option[String], Option[String], Option[String], - Option[String], Option[String]) + Option[String], Option[String], Option[String]) ](insertSql) val values = rows.map { r => @@ -128,7 +129,8 @@ object MetricBatchWriter extends MdcLoggable { Option(r.implementedInVersion), Option(r.verb), r.httpCode, Option(r.correlationId), Option(r.responseBody), Option(r.sourceIp), Option(r.targetIp), Option(r.apiInstanceId), Option(r.consentReferenceId), - Option(r.certificateTrust), Option(r.certificateTrustDetail) + Option(r.certificateTrust), Option(r.certificateTrustDetail), + Option(r.authType) ) } diff --git a/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala b/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala index 6697c04976..bb305ec7d2 100644 --- a/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala +++ b/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala @@ -149,7 +149,7 @@ class ResourceUser extends LongKeyedMapper[ResourceUser] with User with ManyToMa object ResourceUser extends ResourceUser with LongKeyedMetaMapper[ResourceUser]{ // userId_ is deliberately NOT declared here: MigrationOfUserIdIndexes creates a stronger // UNIQUE index on it (resourceuser_userid_unique). CreatedByConsentId is the delegation - // registry — consent-agent fan-down (/my/metrics, /my/banks) and effectiveHumanUserId join + // registry — consent-agent fan-down (/my/metrics, /my/banks) and accountableUserId join // through it; nothing else indexes it, which matters on consent-heavy instances where every // consent mints a user row. override def dbIndexes = UniqueIndex(provider_, providerId) :: Index(CreatedByConsentId) :: super.dbIndexes diff --git a/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala b/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala index 4633114f1b..e0faac79aa 100644 --- a/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala +++ b/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala @@ -227,7 +227,8 @@ object MetricsArchiveScheduler extends MdcLoggable { i.getApiInstanceId(), i.getConsentReferenceId(), i.getCertificateTrust(), - i.getCertificateTrustDetail() + i.getCertificateTrustDetail(), + i.getAuthType() ) } diff --git a/obp-api/src/test/resources/frozen_type_meta_data b/obp-api/src/test/resources/frozen_type_meta_data index a15d8f1900..a49f5f72b3 100644 Binary files a/obp-api/src/test/resources/frozen_type_meta_data and b/obp-api/src/test/resources/frozen_type_meta_data differ diff --git a/obp-api/src/test/resources/frozen_type_meta_data.txt b/obp-api/src/test/resources/frozen_type_meta_data.txt index d7bddd02ce..bf2252ef5b 100644 --- a/obp-api/src/test/resources/frozen_type_meta_data.txt +++ b/obp-api/src/test/resources/frozen_type_meta_data.txt @@ -511,7 +511,6 @@ field code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON.SeverJWK use String field code.api.util.APIUtil.BooleanBody value Boolean field code.api.util.APIUtil.EndpointInfo name String field code.api.util.APIUtil.EndpointInfo version String -field code.api.util.APIUtil.JArrayBody value org.json4s.JArray field code.api.v1_2_1.APIInfoJSON connector String field code.api.v1_2_1.APIInfoJSON git_commit String field code.api.v1_2_1.APIInfoJSON hosted_by code.api.v1_2_1.HostedBy @@ -3145,6 +3144,7 @@ field code.dynamicResourceDoc.JsonDynamicResourceDoc errorResponseBodies String field code.dynamicResourceDoc.JsonDynamicResourceDoc exampleRequestBody Option[org.json4s.JValue] field code.dynamicResourceDoc.JsonDynamicResourceDoc methodBody String field code.dynamicResourceDoc.JsonDynamicResourceDoc partialFunctionName String +field code.dynamicResourceDoc.JsonDynamicResourceDoc programmingLang String field code.dynamicResourceDoc.JsonDynamicResourceDoc requestUrl String field code.dynamicResourceDoc.JsonDynamicResourceDoc requestVerb String field code.dynamicResourceDoc.JsonDynamicResourceDoc roles String diff --git a/obp-api/src/test/resources/kryo_golden_chill_0_9_3.txt b/obp-api/src/test/resources/kryo_golden_chill_0_9_3.txt new file mode 100644 index 0000000000..e61fe9e03a --- /dev/null +++ b/obp-api/src/test/resources/kryo_golden_chill_0_9_3.txt @@ -0,0 +1,15 @@ +# Kryo/chill golden fixture. Written by the chill on the generating classpath -- +# chill 0.9.3 / chill-bijection 0.9.1 from the pre-migration (Scala 2.12) build, +# the versions PR #2890 replaces with 0.9.5. +# One line per value: namebase64(KryoInjection(value)). +# Regenerating this with the NEW chill would defeat its entire purpose. +string AwFhLWNhY2hlZC1zdHJpbuc= +int AlQ= +long CZaT2J/uRw== +boolean BQE= +double CkAKAAAAAAAA +jlist-string AQBqYXZhLnV0aWwuQXJyYXlMaXP0AQMDAYJhAwGCYgMBgmM= +jlist-empty AQBqYXZhLnV0aWwuQXJyYXlMaXP0AQA= +jmap AQBqYXZhLnV0aWwuTGlua2VkSGFzaE1h8AEBAwFrsQMBdrE= +nested AQBqYXZhLnV0aWwuQXJyYXlMaXP0AQIBAAEBAwGCeAEAAQIDAYJ5AwGCeg== +byte-array XwEFAQIDBA== diff --git a/obp-api/src/test/resources/kryo_scala_golden_chill_0_9_3.tsv b/obp-api/src/test/resources/kryo_scala_golden_chill_0_9_3.tsv new file mode 100644 index 0000000000..5af54008cc --- /dev/null +++ b/obp-api/src/test/resources/kryo_scala_golden_chill_0_9_3.tsv @@ -0,0 +1,12 @@ +# name base64(chill 0.9.3 bytes) runtime class as written +scala-list-string dwEDAwGCYQMBgmIDAYJj scala.collection.immutable.$colon$colon +scala-list-empty dgEA scala.collection.immutable.Nil$ +scala-list-int dwEDAgICBAIG scala.collection.immutable.$colon$colon +scala-vector FAECAwGCeAMBgnk= scala.collection.immutable.Vector +scala-map GwECJwEDAWuxAwF2sScBAwFrsgMBdrI= scala.collection.immutable.Map$Map2 +scala-set FgECAwGCYQMBgmI= scala.collection.immutable.Set$Set2 +scala-seq dwECAwFzsQMBc7I= scala.collection.immutable.$colon$colon +scala-option-some EQEDAWhlcuU= scala.Some +scala-option-none dAE= scala.None$ +scala-tuple JwEDAYJhAgI= scala.Tuple2 +scala-nested-list dwECdwEBAwGCeHcBAgMBgnkDAYJ6 scala.collection.immutable.$colon$colon diff --git a/obp-api/src/test/scala/code/api/cache/CacheKeyFormatTest.scala b/obp-api/src/test/scala/code/api/cache/CacheKeyFormatTest.scala new file mode 100644 index 0000000000..2f18bd3eda --- /dev/null +++ b/obp-api/src/test/scala/code/api/cache/CacheKeyFormatTest.scala @@ -0,0 +1,153 @@ +package code.api.cache + +import org.scalatest.{FlatSpec, Matchers} + +import scala.concurrent.duration._ +import scala.jdk.CollectionConverters._ + +/** + * The exact string scalacache derives for a cache key, pinned. + * + * PR #2890 moves scalacache 0.9.3 -> 0.28.0, which is a rewrite rather than an upgrade: the + * backends were restructured, `memoize` became `memoizeF`, `ttl: Duration` became `Some(ttl)`, + * and the Guava store's value type changed. Key derivation lives inside that rewrite. + * + * Why the existing coverage is not enough. InMemoryCachingTest asserts: + * + * InMemory.countKeys(s"*$key*") should equal(1) + * + * which proves the caller's own string survives INTO the derived key -- a substring check. It + * passes for any prefix, any separator, any argument rendering, as long as the caller's string + * is in there somewhere. That is not enough for the thing that actually depends on this format: + * + * NewStyle.scala:3306 Redis.deleteKeysByPattern("*getMethodRoutings*") + * Caching.scala:121 Redis.deleteKeysByPattern(s"${RATE_LIMIT_ACTIVE_PREFIX}${id}_*") + * + * Invalidation is pattern matching over the whole key. If the derivation grows a prefix, changes + * a separator, or renders the enclosing method differently, the cache keeps caching and the + * invalidation quietly stops matching anything -- `deleteKeysByPattern` returns 0 and swallows + * it, so nothing anywhere reports a problem. Stale MethodRoutings then serve for a full TTL. + * + * So this asserts the FULL derived key, read back out of the store, not a substring of it. + * The value is written down rather than computed, because a check that derives its expectation + * the same way the code does cannot fail. + * + * If this test breaks after a scalacache change, the fix is NOT to update the expected string + * until it passes. It is to check every deleteKeysByPattern call site against the new format + * first -- this test failing is that review being demanded, which is its whole purpose. + */ +class CacheKeyFormatTest extends FlatSpec with Matchers { + + private val ttl = 60.seconds + + private def storedKeys: Set[String] = + InMemory.underlyingGuavaCache.asMap().keySet().asScala.toSet + + private def freshMarker(tag: String): String = + s"CacheKeyFormatTest-$tag-${java.util.UUID.randomUUID().toString.take(8)}" + + /** + * The derivation, recorded from the scalacache on this branch. + * + * Discovered by writing this test with the naive expectation (the bare caller key) and reading + * what came back. The wrapper is scalacache's MethodCallToStringConverter: the enclosing + * method's fully-qualified name, then each parameter list rendered in order -- so the caller's + * key arrives inside `Some(...)`, and the two @cacheKeyExclude lists render as empty `()`. + * + * That wrapper is exactly what a substring assertion cannot see, and exactly what an + * invalidation glob has to survive. + */ + private def derivedKey(callerKey: String): String = + s"code.api.cache.InMemory.memoizeSyncWithInMemory(Some($callerKey))()()" + + "the derived cache key" should "be exactly the recorded derivation of the caller's key" in { + val marker = freshMarker("exact") + val before = storedKeys + Caching.memoizeSyncWithImMemory(Some(marker))(ttl)("stored") + val added = storedKeys -- before + + withClue(s"one memoize call should add exactly one key; added=${added.mkString(", ")} ") { + added.size shouldBe 1 + } + + // THE assertion. Not `contains`, not a regex -- the whole string. + // + // Recorded from the scalacache on this branch. Any change to it means every + // deleteKeysByPattern pattern in the codebase has to be re-read against the new shape + // before this line is updated. + withClue(s"the derived key is '${added.head}' but was recorded as '${derivedKey(marker)}'. " + + s"Before changing the expectation, check NewStyle.scala:3306's " + + s"\"*getMethodRoutings*\" and Caching.scala:121/132's rate-limit patterns still " + + s"match the new shape -- deleteKeysByPattern returns 0 and swallows a miss, so a " + + s"broken pattern is silent. ") { + added.head shouldBe derivedKey(marker) + } + } + + it should "keep the pattern MethodRouting invalidation depends on matchable" in { + // The real one. NewStyle.invalidateMethodRoutingCache issues + // deleteKeysByPattern("*getMethodRoutings*"), so a key derived from a caller string + // containing "getMethodRoutings" must be matched by that glob. + val marker = s"(CacheKeyFormatTest,getMethodRoutings,${java.util.UUID.randomUUID().toString.take(8)})" + val before = storedKeys + Caching.memoizeSyncWithImMemory(Some(marker))(ttl)("routings") + val added = (storedKeys -- before).head + + val glob = "*getMethodRoutings*" + val regex = glob.replace("*", ".*") + withClue(s"derived key '$added' is not matched by the invalidation pattern '$glob'. " + + s"NewStyle.invalidateMethodRoutingCache would delete nothing and report nothing. ") { + added.matches(regex) shouldBe true + } + InMemory.countKeys(glob) should be >= 1 + } + + it should "give different callers different keys" in { + // A derivation that collapsed distinct callers onto one key would serve one caller's value + // to another -- and every substring assertion in the suite would still pass. + val a = freshMarker("distinct-a") + val b = freshMarker("distinct-b") + val before = storedKeys + Caching.memoizeSyncWithImMemory(Some(a))(ttl)("value-a") + Caching.memoizeSyncWithImMemory(Some(b))(ttl)("value-b") + val added = storedKeys -- before + + added.size shouldBe 2 + Caching.memoizeSyncWithImMemory(Some(a))(ttl)("recomputed-a") shouldBe "value-a" + Caching.memoizeSyncWithImMemory(Some(b))(ttl)("recomputed-b") shouldBe "value-b" + } + + it should "not let one caller's key be a prefix-collision of another's" in { + // `deleteKeysByPattern` globs. If a key were rendered such that one caller's string is a + // prefix of another's WITHOUT a delimiter, invalidating the first would take out the second. + val short = freshMarker("collide") + val long = s"${short}-extended" + val before = storedKeys + Caching.memoizeSyncWithImMemory(Some(short))(ttl)("short-value") + Caching.memoizeSyncWithImMemory(Some(long))(ttl)("long-value") + val added = storedKeys -- before + + added.size shouldBe 2 + withClue(s"keys: ${added.mkString(", ")} -- an exact-match glob on the shorter key must not " + + s"also match the longer one. ") { + added.count(_ == derivedKey(short)) shouldBe 1 + added.count(_ == derivedKey(long)) shouldBe 1 + } + + // The collision the wrapper actually prevents: because the caller key is ENCLOSED rather + // than concatenated, the `)` that follows it is a hard delimiter -- the shorter key's full + // derivation is not a prefix of the longer one's, so nothing anchored on it can reach the + // longer entry. + // + // Asserted by set membership rather than through countKeys, deliberately. countKeys builds + // its matcher as `pattern.replace("*", ".*").r` (InMemory.scala:49), so every other regex + // metacharacter in the pattern is live -- and a derived key is full of them: `(`, `)` and + // `.` all appear in `...memoizeSyncWithInMemory(Some(x))()()`. Passing a whole derived key + // to countKeys therefore asks a question about regex syntax, not about key collision. + // (That is a real sharp edge in a helper whose callers pass user-shaped strings, but it + // belongs in its own finding rather than being asserted sideways from here.) + withClue(s"the shorter key's derivation must not be a prefix of the longer one's. ") { + derivedKey(long).startsWith(derivedKey(short)) shouldBe false + } + } +} diff --git a/obp-api/src/test/scala/code/api/cache/CacheSerializationNamespaceTest.scala b/obp-api/src/test/scala/code/api/cache/CacheSerializationNamespaceTest.scala new file mode 100644 index 0000000000..af32030f35 --- /dev/null +++ b/obp-api/src/test/scala/code/api/cache/CacheSerializationNamespaceTest.scala @@ -0,0 +1,105 @@ +package code.api.cache + +import code.setup.RedisTestTarget +import org.scalatest.{FlatSpec, Matchers} +import scalacache.{CacheConfig, DefaultCacheKeyBuilder} + +/** + * Two OBP-API versions must not read each other's cached bytes. + * + * The defect this pins is not hypothetical and is not a decode failure. An empty `List`, written + * by chill 0.9.3 on Scala 2.12, decodes under chill 0.9.5 on 2.13 into a + * `scala.collection.immutable.Queue`. The decode SUCCEEDS; the call site, whose signature says + * `List`, is where it dies: + * + * class scala.collection.immutable.Queue cannot be cast to + * class scala.collection.immutable.List + * + * Measured on `GET /management/dynamic-message-docs` and `GET /management/connector-methods`: + * 200 on 2.12, 500 on 2.13 reading the entry 2.12 wrote, and correct in either version alone. + * The 500 lasts the whole TTL, because a read that throws does not evict the key. A rolling + * upgrade, or any upgrade against a warm Redis, produces exactly this. + * + * `Redis.serializationNamespace` prefixes every memoized key with the Scala binary version and a + * manually bumpable counter, so entries from another build are not addressable at all and expire + * on their own. + * + * ── What is asserted ── + * + * The PROPERTY, not the string. Asserting the current prefix would pin `obpser1-scala2.13`, which + * says nothing about whether isolation holds and turns every legitimate bump into a test edit. + * What has to stay true is that two different namespaces cannot see each other's entries, and + * that one namespace still sees its own -- an isolation that isolated everything, including a + * version from itself, would "pass" while disabling the cache entirely. + * + * These run against a real Redis. RedisTestTarget cancels them when none is reachable, and + * OBP_TEST_REDIS_REQUIRED=true turns that cancel into a failure so CI cannot lose the check + * silently. + */ +class CacheSerializationNamespaceTest extends FlatSpec with Matchers { + + /** A stand-in caller key; its value is arbitrary, only its stability across calls matters. */ + private val SampleCallerKey = "code.example.Provider.getAll(Some(bank))" + + /** This build's namespace, spelled out rather than derived, so a test asserting against it + * fails loudly if the derivation and the literal ever disagree. */ + private val CurrentNamespace = "obpser1-scala2.13" + + private def keyFor(namespace: String, callerKey: String): String = + CacheConfig(cacheKeyBuilder = DefaultCacheKeyBuilder(keyPrefix = Some(namespace))) + .cacheKeyBuilder.toCacheKey(Seq(callerKey)) + + "the derived cache key" should "differ between two serialization namespaces" in { + val a = keyFor("obpser1-scala2.12", SampleCallerKey) + val b = keyFor(CurrentNamespace, SampleCallerKey) + + withClue(s"2.12 key <$a> and 2.13 key <$b> are the same, so a 2.13 instance would read the " + + s"bytes a 2.12 instance wrote -- which is the defect this exists to prevent. ") { + a should not equal b + } + a should include("2.12") + b should include("2.13") + } + + it should "also differ when only the manual counter is bumped" in { + // The Scala version does not move for a dependency upgrade that changes the encoding -- + // chill 0.9.3 to 0.9.5 on its own would not have. The counter is the escape hatch for that, + // and it is only an escape hatch if it actually changes the key. + keyFor(CurrentNamespace, SampleCallerKey) should not equal keyFor("obpser2-scala2.13", SampleCallerKey) + } + + it should "stay stable for one namespace, or nothing would ever be a cache hit" in { + keyFor(CurrentNamespace, SampleCallerKey) shouldBe keyFor(CurrentNamespace, SampleCallerKey) + } + + "the namespace this build uses" should "name the Scala binary version it was compiled against" in { + // Derived, not asserted verbatim: the point is that it tracks the axis that actually moved. + val expected = scala.util.Properties.versionNumberString.split('.').take(2).mkString(".") + val probe = keyFor(s"obpser1-scala$expected", "x") + probe should include(expected) + } + + "a real Redis" should "not return an entry written under a different namespace" in { + RedisTestTarget.requireReachable(Redis.isRedisReady, "the cross-namespace isolation check") + + val caller = s"code.example.Probe.roundTrip(${System.nanoTime()})" + val oldKey = keyFor("obpser1-scalaOLD", caller) + val newKey = keyFor("obpser1-scalaNEW", caller) + + import code.api.JedisMethod + try { + Redis.use(JedisMethod.SET, oldKey, Some(60), Some("written-by-the-other-version")) + + withClue("the new namespace found the old namespace's entry -- the prefix is not isolating ") { + Redis.use(JedisMethod.GET, newKey, None, None) shouldBe None + } + withClue("the old namespace could not read back its OWN entry, so this test proved nothing " + + "about isolation -- it would pass with the cache switched off entirely ") { + Redis.use(JedisMethod.GET, oldKey, None, None) shouldBe Some("written-by-the-other-version") + } + } finally { + Redis.use(JedisMethod.DELETE, oldKey, None, None) + Redis.use(JedisMethod.DELETE, newKey, None, None) + } + } +} diff --git a/obp-api/src/test/scala/code/api/cache/KryoGoldenCompatTest.scala b/obp-api/src/test/scala/code/api/cache/KryoGoldenCompatTest.scala new file mode 100644 index 0000000000..cb5b672f02 --- /dev/null +++ b/obp-api/src/test/scala/code/api/cache/KryoGoldenCompatTest.scala @@ -0,0 +1,226 @@ +package code.api.cache + +import org.scalatest.{FlatSpec, Matchers} + +import java.util.Base64 +import scala.io.Source +import scala.util.Try + +/** + * What the OLD chill wrote, the NEW chill must not silently misread. + * + * PR #2890 moves chill 0.9.3 -> 0.9.5 (and chill-bijection 0.9.1 -> 0.9.5), and a chill upgrade + * carries a Kryo upgrade. The commit that made the move says what that means for a live system: + * + * "the entries already in Redis were written by the old one, so some of them will fail to + * decode after the rollout. A test environment never shows this, because it starts from an + * empty cache." + * + * That last sentence is the problem this file exists for. The only other Kryo test in the suite, + * RedisDeserializeMissTest, round-trips through `encode` and `decode` -- BOTH of which run on + * whichever chill is on the classpath. A format change is invisible to it by construction: the + * new encoder and the new decoder agree with each other no matter what they agree on. + * + * So the fixtures had to be produced from outside the current build. Two of them, both encoded on + * the pre-migration 2.12 classpath by the real chill 0.9.3, neither regenerable once every + * checkout carries 0.9.5: + * + * kryo_golden_chill_0_9_3.txt ten Java values namebase64 + * kryo_scala_golden_chill_0_9_3.tsv eleven Scala values namebase64runtime class + * + * ── Why the second fixture, and why it records a class ── + * + * The first version of this file held Java collections only and compared with `==`. It could not + * have caught the defect it was written to catch, for two independent reasons, and both were + * found the hard way -- by the defect reaching a running instance. + * + * What OBP-API actually caches is Scala collections; `java.util.ArrayList` appears nowhere in + * the memoized providers. And under `==` a Scala `List()` EQUALS a `Queue()`: both are `Seq`, and + * Seq equality is element-wise, so an empty one of each compares equal. An empty `List` written + * by 0.9.3 decodes under 0.9.5 into a `scala.collection.immutable.Queue`, which the old assertion + * would have waved through -- while every call site whose signature says `List` fails with + * + * class scala.collection.immutable.Queue cannot be cast to + * class scala.collection.immutable.List + * + * Measured on GET /management/dynamic-message-docs and GET /management/connector-methods: 200 on + * 2.12, 500 on 2.13 reading 2.12's entry, for the whole TTL, because a read that throws does not + * evict the key. + * + * So the Scala fixture records the runtime class each value was written as, and this file asserts + * on THAT. Equality is not enough; the class is what the call site depends on. + * + * ── The four outcomes ── + * + * decodes to the same value, same class fine + * fails to decode fine -- a cold cache, which the upgrade note accepts + * decodes to a DIFFERENT value asserted against from the start + * decodes to an equal value of another CLASS the one that got through, now asserted + * + * Survival counts are reported rather than bounded: how many of the values survive is a fact + * about two third-party libraries, not something this branch controls, and an assertion on it + * would freeze a number nobody could act on. What the run is for is the last two outcomes. + */ +class KryoGoldenCompatTest extends FlatSpec with Matchers { + + private val JAVA_FIXTURE = "/kryo_golden_chill_0_9_3.txt" + private val SCALA_FIXTURE = "/kryo_scala_golden_chill_0_9_3.tsv" + + private def readFixture(path: String): List[Array[String]] = { + val stream = getClass.getResourceAsStream(path) + stream should not be null + val src = Source.fromInputStream(stream, "UTF-8") + try src.getLines() + .filterNot(l => l.trim.isEmpty || l.startsWith("#")) + .map(_.split("\t")) + .toList + finally src.close() + } + + /** name -> the bytes chill 0.9.3 produced for it. */ + private lazy val javaGolden: List[(String, Array[Byte])] = + readFixture(JAVA_FIXTURE).map(f => f(0) -> Base64.getDecoder.decode(f(1))) + + /** name -> (bytes, the runtime class the value had WHEN WRITTEN). */ + private lazy val scalaGolden: List[(String, Array[Byte], String)] = + readFixture(SCALA_FIXTURE).map(f => (f(0), Base64.getDecoder.decode(f(1)), f(2))) + + /** The values the Java bytes are supposed to mean. Written out here, not derived. */ + private val expected: Map[String, Any] = Map( + "string" -> "a-cached-string", + "int" -> 42, + "long" -> 1234567890123L, + "boolean" -> true, + "double" -> 3.25d, + "jlist-string" -> java.util.Arrays.asList("a", "b", "c"), + "jlist-empty" -> new java.util.ArrayList[String](), + "jmap" -> { val m = new java.util.LinkedHashMap[String, String](); m.put("k1", "v1"); m }, + "nested" -> java.util.Arrays.asList( + java.util.Arrays.asList("x"), + java.util.Arrays.asList("y", "z")), + "byte-array" -> Array[Byte](1, 2, 3, 4) + ) + + private def sameValue(a: Any, b: Any): Boolean = (a, b) match { + case (x: Array[_], y: Array[_]) => x.sameElements(y) + case (x, y) => x == y + } + + // ── fixtures present ─────────────────────────────────────────────────────────────── + + "both fixtures" should "be present and non-trivial" in { + // A fixture that failed to load would make every assertion below vacuous. + withClue("kryo_golden_chill_0_9_3.txt is missing or empty -- without it this file asserts " + + "nothing at all. It cannot be regenerated from this branch; recover it from git. ") { + javaGolden.size should be >= 8 + } + withClue("kryo_scala_golden_chill_0_9_3.tsv is missing or empty. This is the fixture that " + + "covers what OBP-API actually caches; without it the Java values alone would pass " + + "while the Scala ones drift, which is exactly what happened once already. ") { + scalaGolden.size should be >= 8 + } + scalaGolden.map(_._1).toSet should contain allOf ("scala-list-empty", "scala-map", "scala-option-none") + } + + // ── the assertion the first version was missing ──────────────────────────────────── + + /** + * Class drift that is known, and the reason it can no longer reach a caller. + * + * A signed-off baseline rather than a hard zero, for the same reason the contract suite keeps + * pr90-base.accepted.json: this is a property of two third-party libraries, not something this + * branch can change, and a permanently red suite is a suite people learn to ignore. What must + * stay red is drift that nobody has looked at -- so anything NOT listed here fails, and adding + * a line means writing down why it is safe. + */ + private val knownDrift: Map[String, String] = Map( + "scala-list-empty" -> + ("Nil$ decodes as Queue under chill 0.9.5. Mitigated by Redis.serializationNamespace: the " + + "cache key carries the Scala binary version, so a 2.13 instance cannot address the entry " + + "a 2.12 instance wrote and it expires on its own TTL. CacheSerializationNamespaceTest " + + "pins that isolation; remove it and this becomes reachable again.") + ) + + it should "never decode a Scala value into a DIFFERENT runtime class, except where recorded" in { + import com.twitter.chill.KryoInjection + + val drifted = scalaGolden.flatMap { case (name, bytes, writtenAs) => + KryoInjection.invert(bytes).toOption.flatMap { v => + val nowIs = if (v == null) "null" else v.getClass.getName + // Subclassing is not drift: a Vector written as `Vector` and read back as `Vector1` is + // still assignable to every signature that named Vector, and nothing at a call site can + // tell. What breaks is a class that is merely EQUAL -- List() == Queue() is true, and a + // `List` signature still throws ClassCastException on it. + val assignable = + try Class.forName(writtenAs).isInstance(v) catch { case _: Throwable => nowIs == writtenAs } + if (assignable) None + else Some(s"$name: written as <$writtenAs>, decodes under this chill as <$nowIs>" + + (if (v.isInstanceOf[Iterable[_]]) " -- equal by value, so an == comparison " + + "would call this correct while every call site declaring the original type " + + "fails with ClassCastException" else "")) + } + } + + val unexplained = drifted.filterNot(line => knownDrift.keys.exists(k => line.startsWith(k + ":"))) + drifted.foreach { line => + knownDrift.collectFirst { case (k, why) if line.startsWith(k + ":") => + info(s"known drift -- $line") + info(s" mitigation: $why") + } + } + + withClue(s"${unexplained.size} Scala value(s) drift into a class the original signature cannot " + + s"hold, and are not in knownDrift. This is not a cold cache -- the read SUCCEEDS and " + + s"the caller gets a ClassCastException for the whole TTL, with nothing in any log to " + + s"say so. Either mitigate it or add it to knownDrift with the reason it cannot reach " + + s"a caller:\n${unexplained.mkString("\n")}\n") { + unexplained shouldBe empty + } + + // The baseline must not outlive what it describes. A name listed here that no longer drifts + // is a line nobody will delete, and the next reader takes it as still true. + val staleEntries = knownDrift.keys.filterNot(k => drifted.exists(_.startsWith(k + ":"))).toList + withClue(s"knownDrift lists ${staleEntries.mkString(", ")}, which no longer drift. Remove " + + s"them, or the baseline documents a hazard that stopped existing. ") { + staleEntries shouldBe empty + } + } + + it should "never decode old bytes into a DIFFERENT value" in { + import com.twitter.chill.KryoInjection + + val misread = javaGolden.flatMap { case (name, bytes) => + KryoInjection.invert(bytes) match { + case scala.util.Success(v) if !sameValue(v, expected(name)) => + Some(s"$name: old bytes decoded to <$v> (${v.getClass.getName}) but were written as " + + s"<${expected(name)}> (${expected(name).getClass.getName})") + case _ => None // decoded correctly, or failed -- both acceptable, see the header + } + } + + withClue(s"${misread.size} value(s) written by chill 0.9.3 decode under this chill into " + + s"something OTHER than what was written:\n${misread.mkString("\n")}\n") { + misread shouldBe empty + } + } + + it should "report how much of an existing cache survives the upgrade" in { + import com.twitter.chill.KryoInjection + + val (jOk, jFailed) = javaGolden.partition { case (name, bytes) => + Try(KryoInjection.invert(bytes)).toOption.flatMap(_.toOption).exists(sameValue(_, expected(name))) + } + val (sOk, sFailed) = scalaGolden.partition { case (_, bytes, writtenAs) => + KryoInjection.invert(bytes).toOption.exists(v => + try Class.forName(writtenAs).isInstance(v) catch { case _: Throwable => false }) + } + // Informational on purpose -- see the header. The rollout consequence of a failure is a + // recompute, which is a cost rather than a defect, and pinning the number would freeze a + // property of two third-party libraries. + info(s"java ${jOk.size}/${javaGolden.size} values written by chill 0.9.3 still decode correctly") + info(s"scala ${sOk.size}/${scalaGolden.size} values still decode into an assignable class") + if (jFailed.nonEmpty) info(s"cold on rollout (java): ${jFailed.map(_._1).mkString(", ")}") + if (sFailed.nonEmpty) info(s"cold on rollout (scala): ${sFailed.map(_._1).mkString(", ")}") + succeed + } +} diff --git a/obp-api/src/test/scala/code/api/cache/MethodRoutingCacheInvalidationTest.scala b/obp-api/src/test/scala/code/api/cache/MethodRoutingCacheInvalidationTest.scala index f3d759b45b..3856ccac5d 100644 --- a/obp-api/src/test/scala/code/api/cache/MethodRoutingCacheInvalidationTest.scala +++ b/obp-api/src/test/scala/code/api/cache/MethodRoutingCacheInvalidationTest.scala @@ -4,6 +4,8 @@ import java.util.UUID import org.scalatest.{FlatSpec, Matchers} +import code.setup.RedisTestTarget + import scala.concurrent.duration._ /** @@ -26,7 +28,7 @@ class MethodRoutingCacheInvalidationTest extends FlatSpec with Matchers { Caching.memoizeSyncWithProvider(Some(cacheKey))(ttl)(f) "deleteKeysByPattern(*getMethodRoutings*)" should "invalidate memoized entries so the next read recomputes" in { - assume(Redis.isRedisReady, "requires a reachable Redis") + RedisTestTarget.requireReachable(Redis.isRedisReady, "the MethodRouting cache checks") val marker = s"inv-${UUID.randomUUID().toString}" val cacheKey = s"(MethodRoutingCacheInvalidationTest,getMethodRoutings,$marker)" var computations = 0 @@ -44,7 +46,7 @@ class MethodRoutingCacheInvalidationTest extends FlatSpec with Matchers { } "a corrupted cache entry" should "behave as a miss: recompute once and repopulate with valid bytes" in { - assume(Redis.isRedisReady, "requires a reachable Redis") + RedisTestTarget.requireReachable(Redis.isRedisReady, "the MethodRouting cache checks") val marker = s"poison-${UUID.randomUUID().toString}" val cacheKey = s"(MethodRoutingCacheInvalidationTest,getMethodRoutings,$marker)" var computations = 0 diff --git a/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala b/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala new file mode 100644 index 0000000000..6d1a29c275 --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/AuthSweepTest.scala @@ -0,0 +1,288 @@ +package code.api.sweep + +import cats.effect.IO +import cats.effect.unsafe.IORuntime +import code.api.util.APIUtil.ResourceDoc +import code.api.util.ErrorMessages.{ApplicationNotIdentified, AuthenticatedUserIsRequired, UserHasMissingRoles} +import code.api.util.http4s.Http4sApp +import code.setup.{DefaultUsers, ServerSetupWithTestData} +import fs2.Stream +import org.http4s.{Header, Headers, Method, Request, Uri} +import org.json4s.JValue +import org.json4s.JsonAST.JObject +import org.scalatest.Tag +import org.typelevel.ci.CIString +import com.openbankproject.commons.util.JsonAliases.parse + +/** + * Every endpoint answers an unauthenticated call the way its own ResourceDoc says it will. + * + * Why a sweep instead of more hand-written suites: of the 870 endpoints a caller can reach, + * 384 are referenced by no test at all, and of the ones that ARE tested only about a third + * carry an anonymous-access scenario. Writing those by hand is several hundred near-identical + * files that then rot one endpoint at a time; driving them off the registry means an endpoint + * added tomorrow is swept the day it is registered, and SweepCoverageTest fails if it is not. + * + * Three assertions, chosen by what the doc declares: + * + * auth required, no roles anonymous -> 401 with exactly AuthenticatedUserIsRequired + * auth required, roles anonymous -> 401; authenticated-without-the-role -> 403 + * public anonymous -> anything EXCEPT 401 + * + * The public case is deliberately weak. A public endpoint may well answer 400 or 404 to a call + * with no body and a nonexistent id — that is not an authentication defect, and asserting 200 + * would make this sweep a fixture problem instead of an auth check. What must never happen is a + * public endpoint demanding credentials, and that is what is asserted. + * + * The 403 assertion uses startWith, not equal: at runtime the message carries the missing roles + * joined with " or ", and ApiRole.requiresBankId appends " for BankId(...)". An equality + * assertion here passes locally and fails the moment an endpoint gains a second role. + * + * ── Why one scenario per version rather than per endpoint ── + * ServerSetupWithTestData.beforeEach wipes and rebuilds banks, accounts and views for EVERY + * scenario. At ~1600 assertions that fixture cost, not the assertions, would dominate: the + * requests themselves run in-process against Http4sApp.httpApp with no TCP and no server, and + * cost single-digit milliseconds each. So each scenario sweeps one version, collects every + * mismatch, and fails once with the whole list. The per-endpoint detail that a + * scenario-per-endpoint layout would have given is preserved in that list — each line names the + * operationId, the verb, the URL, the expectation and what actually came back. + */ +object AuthSweepTest { + + /** + * The single definition of what this sweep covers. Exposed so SweepCoverageTest's drift check + * reads this directly instead of re-deriving its own copy of the same filter -- two copies of + * one expression are equal by construction and can never catch this sweep's own filtering + * changing independently of FailureSweepTest's. + */ + def scope: List[ResourceDoc] = EndpointCatalog.all.filter(EndpointCatalog.skipReason(_).isEmpty) +} + +class AuthSweepTest extends ServerSetupWithTestData with DefaultUsers with SweepFixtures { + + object AuthSweep extends Tag("AuthSweep") + + implicit val runtime: IORuntime = IORuntime.global + private lazy val app = Http4sApp.httpApp + + /** One in-process request. No TCP, no server startup. */ + private def call(verb: String, path: String, headers: Map[String, String]): (Int, JValue) = { + val method = Method.fromString(verb.toUpperCase).getOrElse(Method.GET) + val req = Request[IO]( + method = method, + uri = Uri.unsafeFromString(path), + headers = Headers(headers.map { case (k, v) => Header.Raw(CIString(k), v) }.toList), + body = Stream.empty + ) + val resp = app.run(req).unsafeRunSync() + val bodyStr = resp.bodyText.compile.string.unsafeRunSync() + val json = try { if (bodyStr.trim.isEmpty) JObject(Nil) else parse(bodyStr) } + catch { case _: Exception => JObject(Nil) } + (resp.status.code, json) + } + + private def messageOf(json: JValue): String = { + implicit val formats = code.api.util.CustomJsonFormats.formats + (json \ "message").extractOpt[String].getOrElse("") + } + + /** A token for a user holding no entitlements at all — the natural 403 probe. */ + private def noRoleHeaders: Map[String, String] = Map("DirectLogin" -> s"token=${token1.value}") + + /** + * Real identifiers from the fixtures, for the role assertion only. + * + * An endpoint that declares BankNotFound and carries BANK_ID validates the bank before it + * checks roles, so a nonexistent bank answers 404 and the role gate never runs. These come + * from the fixture banks/accounts ServerSetupWithTestData creates, read directly rather than + * over HTTP — the sweep is in-process and a round trip per lookup would be the only slow part + * of it. + */ + private lazy val realEntities: Map[String, String] = realBankId match { + case Some(bankIdValue) => + val accountId = code.model.dataAccess.MappedBankAccount + .find(net.liftweb.mapper.By(code.model.dataAccess.MappedBankAccount.bank, bankIdValue)) + .map(_.accountId.value) + Map("BANK_ID" -> bankIdValue) ++ accountId.map("ACCOUNT_ID" -> _).toList.toMap + case None => Map.empty + } + + private def describe(doc: ResourceDoc): String = + s"${doc.operationId} ${doc.requestVerb} ${EndpointCatalog.concretePath(doc)}" + + // ── the three checks, each returning a failure line or None ────────────────── + + /** + * Deviations that are deliberate, with the reason each one is not a defect. + * + * A signed-off list rather than a hard zero, for the same reason KryoGoldenCompatTest keeps + * knownDrift: a permanently red suite is one people learn to ignore, and the two entries here + * are both behaviour somebody chose and wrote down. Anything NOT listed still fails, and + * adding a line costs a written justification. + */ + private val expectedAuthDeviation: Map[String, String] = Map( + "OBPv4.0.0-verifyRequestSignResponse" -> + ("Refuses with OBP-20311 'The Request is not signed' -- JWS request signing, a third " + + "authentication mechanism alongside user and application. ResourceDoc has no way to " + + "declare it: authMode covers user/application only, so neither the doc nor this sweep " + + "can express the requirement. The 401 is correct; only the message differs."), + "OBPv4.0.0-createTransactionRequestFreeForm" -> + ("Answers 400 InsufficientAuthorisationToCreateTransactionRequest rather than 403. The " + + "endpoint deliberately does no upfront view/role check and delegates the decision to " + + "checkAuthorisationToCreateTransactionRequest inside the connector -- its own comment " + + "says so, and an existing test depends on it. Whether an authorisation failure ought to " + + "be 400 at all is a product question, not something to change from inside a sweep.") + ) + + /** Which exemptions were actually needed this run -- see the stale-entry scenario below. */ + private val deviationsUsed = java.util.concurrent.ConcurrentHashMap.newKeySet[String]() + + private def deviationFor(doc: ResourceDoc): Option[String] = { + val why = expectedAuthDeviation.get(doc.operationId) + if (why.isDefined) deviationsUsed.add(doc.operationId) + why + } + + private def checkAnonymousIs401(doc: ResourceDoc): Option[String] = { + val (code, json) = call(doc.requestVerb, EndpointCatalog.concretePath(doc), Map.empty) + if (code != 401) + Some(s"${describe(doc)} -- expected 401 for an anonymous call, got $code") + else if (messageOf(json) != AuthenticatedUserIsRequired) + deviationFor(doc) match { + case Some(why) => + info(s"${describe(doc)} -- 401 with '${messageOf(json)}'; expected deviation: $why") + None + case None => + Some(s"${describe(doc)} -- 401 but message was '${messageOf(json)}', expected '$AuthenticatedUserIsRequired'") + } + else None + } + + /** + * A doc that asks for no USER may still ask for an APPLICATION, and that is not a defect. + * + * OBP has three ways to refuse an anonymous caller, and this check originally modelled one: + * + * OBP-20001 User not logged in -- user authentication + * OBP-20200 The application cannot be identified -- consumer/application authentication + * OBP-20311 The Request is not signed -- JWS request signing + * + * `EndpointCatalog.needsAuthentication` reproduces the middleware's predicate, which reads + * only errorResponseBodies and roles -- both about the user. So an endpoint that requires a + * consumer is classified "public" here and then fails this assertion for doing exactly what + * its doc says. Measured on createConsentRequest, getConsentRequest and + * createVRPConsentRequest: all three answer OBP-20200, and the last one spells it out in its + * own description -- "Client, Consumer or Application Authentication is mandatory for this + * endpoint". Their docs were right; this check was wrong. + * + * So a 401 is only a violation when it is the USER one. An application-auth 401 is reported + * as an observation instead of a failure -- named, not silently swallowed, because the doc + * still has no machine-readable way to say "needs an application" unless someone sets + * authMode, and a reader of resource-docs cannot tell. + */ + private def checkPublicIsNot401(doc: ResourceDoc): Option[String] = { + val (code, json) = call(doc.requestVerb, EndpointCatalog.concretePath(doc), Map.empty) + val msg = messageOf(json) + if (code != 401) None + else if (msg.startsWith(ApplicationNotIdentified.take(9))) { + info(s"${describe(doc)} -- declares no user authentication and requires an APPLICATION " + + s"instead ($msg). The doc is accurate about the user; consider authMode = " + + s"ApplicationOnly so resource-docs can say so too.") + None + } else + Some(s"${describe(doc)} -- declares no authentication requirement yet answered 401 " + + s"anonymously with '$msg'") + } + + private def checkNoRoleIs403(doc: ResourceDoc): Option[String] = { + val path = EndpointCatalog.concretePath(doc, realEntities) + val (code, json) = call(doc.requestVerb, path, noRoleHeaders) + val roles = doc.roles.getOrElse(Nil).map(_.toString).mkString(",") + if (code != 403) + deviationFor(doc) match { + case Some(why) => + info(s"${doc.operationId} answered $code rather than 403; expected deviation: $why") + None + case None => + Some(s"${doc.operationId} ${doc.requestVerb} $path -- roles $roles: " + + s"expected 403 for a user holding no entitlements, got $code") + } + else if (!messageOf(json).startsWith(UserHasMissingRoles)) + Some(s"${doc.operationId} ${doc.requestVerb} $path -- 403 but message was " + + s"'${messageOf(json)}', expected it to start with '$UserHasMissingRoles'") + else None + } + + // ── the sweep, one scenario per version ───────────────────────────────────── + + private lazy val byVersion: Map[String, List[ResourceDoc]] = + AuthSweepTest.scope.groupBy(_.implementedInApiVersion.toString) + + feature("Every reachable endpoint enforces the authentication its ResourceDoc declares") { + + byVersion.keys.toList.sorted.foreach { version => + scenario(s"$version -- anonymous calls are refused, public ones are not", AuthSweep) { + // Endpoint-level enable/disable props are read per request by ResourceDocMiddleware, + // and a disabled endpoint falls through to 404 rather than 401 -- which would read as a + // sweep failure. Cleared the way SwaggerDocsTest does; PropsReset restores afterwards. + // Written out in each scenario rather than shared in a helper because + // check_test_isolation.py scans statically: any setPropsValues outside a scenario body + // reads to it as a class-body push, `def` or not. + setPropsValues("api_disabled_endpoints" -> "[]", "api_enabled_endpoints" -> "[]") + val docs = byVersion(version) + + When(s"every one of the ${docs.size} $version endpoints is called with no credentials") + val failures = docs.flatMap { doc => + if (EndpointCatalog.needsAuthentication(doc)) checkAnonymousIs401(doc) + else checkPublicIsNot401(doc) + } + + Then("each one answers as its own ResourceDoc declares") + withClue(s"${failures.size} of ${docs.size} $version endpoints disagreed with their own " + + s"ResourceDoc:\n${failures.mkString("\n")}\n") { + failures shouldBe empty + } + } + } + + byVersion.keys.toList.sorted.foreach { version => + lazy val roleGated = byVersion(version) + .filter(EndpointCatalog.isRoleGated) + .filter(EndpointCatalog.roleSkipReason(_).isEmpty) + + if (roleGated.nonEmpty) { + scenario(s"$version -- role-gated endpoints refuse a user holding no entitlements", AuthSweep) { + setPropsValues("api_disabled_endpoints" -> "[]", "api_enabled_endpoints" -> "[]") + + When(s"every one of the ${roleGated.size} role-gated $version endpoints is called as a user with no entitlements") + val failures = roleGated.flatMap(checkNoRoleIs403) + + Then("each one answers 403 naming the roles it wanted") + withClue(s"${failures.size} of ${roleGated.size} role-gated $version endpoints did not " + + s"refuse an unentitled user:\n${failures.mkString("\n")}\n") { + failures shouldBe empty + } + } + } + } + + // Declared after both version loops, so deviationsUsed is complete when it runs. + scenario("no expectedAuthDeviation entry outlives the behaviour it excuses", AuthSweep) { + import scala.jdk.CollectionConverters._ + val used = deviationsUsed.asScala.toSet + val stale = expectedAuthDeviation.keySet -- used + val unknown = expectedAuthDeviation.keySet -- EndpointCatalog.all.map(_.operationId).toSet + + withClue(s"these endpoints no longer deviate, so their exemption is a claim that stopped " + + s"being true and the next reader will take it as still true: ${stale.mkString(", ")}. " + + s"Delete the entry. ") { + stale shouldBe empty + } + withClue(s"these operationIds are not in the catalog at all -- renamed or removed, and " + + s"the exemption was left behind: ${unknown.mkString(", ")}. ") { + unknown shouldBe empty + } + info(s"${used.size} deviation(s) exercised: ${used.mkString(", ")}") + } + } +} diff --git a/obp-api/src/test/scala/code/api/sweep/EndpointCatalog.scala b/obp-api/src/test/scala/code/api/sweep/EndpointCatalog.scala new file mode 100644 index 0000000000..c5bba1dd2a --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/EndpointCatalog.scala @@ -0,0 +1,163 @@ +package code.api.sweep + +// EndpointAuthMode and its four cases are members of the APIUtil object, not of the package. +import code.api.util.APIUtil.{ResourceDoc, UserOnly} +import code.api.util.ErrorMessages.$AuthenticatedUserIsRequired +import code.api.util.ApiTag + +/** + * The one place that answers "what endpoints exist, and what does each one claim about auth". + * + * Every sweep in this package reads the catalog from here rather than assembling its own, so + * that the coverage identity in SweepCoverageTest is checkable: asserted + skipped == catalog, + * with no third bucket anyone can quietly slip an endpoint into. + * + * Three things about the source data are easy to get wrong, and all three are load-bearing: + * + * 1. The docs live on the Http4s objects, NOT on APIMethods*. Every APIMethods{121..600}.scala + * is now a stub whose entire body is `val ImplementationsX = Http4sX.ImplementationsX` — the + * Lift registrations below it are commented out. Reading those files for a catalog finds + * nothing. Http4s700.allResourceDocs is the aggregate: every version's docs, deduplicated by + * (requestUrl, requestVerb) keeping the newest, which is exactly the set a caller can reach. + * + * 2. "Needs authentication" is derived, not declared. There is no flag on ResourceDoc; the + * middleware's own predicate is + * errorResponseBodies.contains($AuthenticatedUserIsRequired) || roles.exists(_.nonEmpty) + * (ResourceDocMiddleware.needsAuthentication). We reproduce it here rather than approximate it. + * + * 3. That predicate has to be evaluated on the CONSTRUCTED ResourceDoc, never on the source text. + * The constructor rewrites errorResponseBodies: it appends AuthenticatedUserIsRequired and + * UserHasMissingRoles when roles are present, and adds/removes AuthenticatedUserIsRequired + * based on the description. Several docs also compute their error list from a prop — e.g. + * getApiProduct branches on getApiProductsIsPublic — so the answer depends on the props in + * force at the moment the sweep runs, which is another reason to ask the object and not a grep. + */ +object EndpointCatalog { + + /** Every endpoint a caller can reach, newest version of each (url, verb). */ + def all: List[ResourceDoc] = code.api.v7_0_0.Http4s700.allResourceDocs.toList + + /** The middleware's own rule, reproduced. See ResourceDocMiddleware.needsAuthentication. */ + def needsAuthentication(doc: ResourceDoc): Boolean = + doc.errorResponseBodies.contains($AuthenticatedUserIsRequired) || doc.roles.exists(_.nonEmpty) + + def isRoleGated(doc: ResourceDoc): Boolean = doc.roles.exists(_.nonEmpty) + + /** + * Why an endpoint is not swept. Every exclusion is one of these — a sweep may not invent a + * reason inline, because SweepCoverageTest counts these and nothing else. + */ + sealed abstract class SkipReason(val why: String) + case object DynamicDoc extends SkipReason( + "tagged apiTagDynamic: created per-database, so its presence is machine state, not contract") + case object NonUserAuthMode extends SkipReason( + "authMode is not UserOnly: an anonymous call may legitimately not be 401 " + + "(ApplicationOnly drops AuthenticatedUserIsRequired entirely). EndpointAuthModeTest covers these.") + case object AutoValidateRolesOff extends SkipReason( + "disableAutoValidateRoles(): roles stay in the doc for the catalog but the framework does " + + "not enforce them, so asserting 403 would assert something no code promises") + + /** Skip reason, or None when the endpoint is in scope for the auth sweep. */ + def skipReason(doc: ResourceDoc): Option[SkipReason] = + if (doc.tags.contains(ApiTag.apiTagDynamic)) Some(DynamicDoc) + else if (doc.authMode != UserOnly) Some(NonUserAuthMode) + else None + + /** Skip reason for the role dimension specifically — a superset of skipReason. */ + def roleSkipReason(doc: ResourceDoc): Option[SkipReason] = + skipReason(doc).orElse( + if (!doc.isAutoValidateRoles) Some(AutoValidateRolesOff) else None) + + /** + * Not every ALL_CAPS segment in a requestUrl is a placeholder. OBP serves real literals in + * that shape — `/transaction-request-types/SANDBOX_TAN/`, `/my/consents/EMAIL` — and + * substituting those produces a URL that routes nowhere, which reads as a 404/400 "auth + * failure" that is entirely the sweep's own doing. The first run of AuthSweepTest hit exactly + * that on nine endpoints across v2.1.0 and v3.1.0. + * + * The production rule lives in ResourceDocMatcher.isTemplateVariable, which consults a private + * `literalAllCapsSegments` set. Copying that set here would give us a second copy to keep in + * sync, and a stale copy fails in the direction that is hardest to notice — a literal newly + * added there would be substituted here and the sweep would quietly stop covering that path. + * + * So this asks the opposite question: rather than "is it a literal", "do I have a value for + * it". A segment is substituted only when its NAME says it is an id or a code. That happens to + * separate the two sets exactly, including the pairs that differ by suffix alone — CARD and + * ACCOUNT are literals, CARD_ID and ACCOUNT_ID are placeholders — and it needs no maintenance + * when a new literal appears, because an unrecognised segment is left alone by default. + */ + /** True when the URL carries at least one segment concretePath would substitute. */ + def hasPlaceholder(doc: ResourceDoc): Boolean = doc.requestUrl.split("/").exists(isPlaceholder) + + private def isPlaceholder(seg: String): Boolean = + seg.nonEmpty && + seg == seg.toUpperCase && + seg.forall(c => c.isLetter || c == '_' || c.isDigit) && + // `ID` rather than `_ID`: the UK Open Banking paths spell them without the separator + // (CONSENTID, ACCOUNTID, BASKETID, DOMESTICPAYMENTID …), and leaving those literal sent + // the string "CONSENTID" to the server as though it were an id. + (seg.endsWith("ID") || seg.endsWith("_CODE") || seg.endsWith("_NAME") || + seg == "PROVIDER" || seg == "USERNAME" || seg == "USER_EMAIL" || + // Named individually because none of the three ends in ID/_CODE/_NAME, yet each is a + // genuine enumerated-value placeholder a live endpoint validates inline, not a literal: + // Http4sBGv2PIS's payment-service branches guard on + // Set("payments","bulk-payments","periodic-payments").contains(paymentService), and + // Http4s310/Http4s400's auth-context-updates and consent SCA branches guard on + // List(StrongCustomerAuthentication.SMS, EMAIL[, IMPLICIT]).contains(scaMethod). Left as + // the literal strings "PAYMENT_SERVICE"/"SCA_METHOD", both guards fail and the sweep + // reports the resulting 404/400 as the endpoint's own defect -- confirmed reproducing + // today for SCA_METHOD via OBPv5.0.0-createUserAuthContextUpdateRequest. + // PAYMENT_PRODUCT is not independently validated anywhere it appears, but is named here + // for the same reason PROVIDER/USERNAME/USER_EMAIL are: it identifies a Berlin Group + // payment product, not a literal segment, and treating it as one just because nothing + // currently checks its value the way PAYMENT_SERVICE and SCA_METHOD are checked would be + // an accident of the present call sites, not the actual contract of the segment. + seg == "PAYMENT_SERVICE" || seg == "PAYMENT_PRODUCT" || seg == "SCA_METHOD") + + // Checked against Http4sSupport's literalAllCapsSegments: not one of the sixteen ends in ID, + // _CODE or _NAME, so the rule above separates the two sets cleanly. + // + // CARDANO, MOBILE_WALLET and ETH_SEND_TRANSACTION are the remaining literals this list does + // not carry, and they stay verbatim on purpose -- substituting over them would route to the + // wrong case entirely (or none), which is the coverage hole this whole heuristic exists to + // avoid on the literal side. + + /** + * The concrete path to call. + * + * `entities` supplies values for the identifiers the caller wants resolvable. Anything not + * named there gets a well-formed value that does not exist. + * + * The distinction matters for the 403 assertion. A ResourceDoc that declares BankNotFound and + * carries BANK_ID is validated for bank existence BEFORE its roles are checked + * (APIUtil.ResourceDoc's isNeedCheckBank), so a nonexistent bank answers 404 and the role gate + * is never reached. The sweep's first run read those 404s as missing 403s across some thirty + * endpoints; they were the entity check doing its job. Passing a real bank id is what makes + * the assertion actually about roles. + */ + def concretePath(doc: ResourceDoc, entities: Map[String, String] = Map.empty): String = { + val segments = doc.requestUrl.split("/").map { seg => + if (isPlaceholder(seg)) entities.getOrElse(seg, defaultValue(seg)) else seg + } + "/obp/" + doc.implementedInApiVersion.apiShortVersion + segments.mkString("/") + } + + /** Well-formed, and deliberately not present. */ + private def defaultValue(seg: String): String = seg match { + case "ACCOUNT_ID" | "USER_ID" => "00000000-0000-0000-0000-000000000000" + // GRANT_VIEW_ID alongside VIEW_ID, and for the same reason a real bank id is passed for the + // role assertion: an endpoint that resolves the view before it checks roles answers 404 for a + // view that does not exist, and the role gate never runs. Measured on + // createTransactionRequestFreeForm, which the sweep reported as "expected 403, got 500" -- + // two defects stacked, the endpoint's raw throw AND this placeholder never resolving. + case "VIEW_ID" | "GRANT_VIEW_ID" => "owner" + case "USER_EMAIL" => "sweep-no-such-user@example.com" + // Enumerated values a live endpoint validates inline (see isPlaceholder) -- a well-formed + // but nonexistent value here would still fail that inline check, same as an id that does not + // exist fails a lookup, so these get one of the endpoint's own accepted values instead. + case "PAYMENT_SERVICE" => "payments" + case "PAYMENT_PRODUCT" => "sepa-credit-transfers" + case "SCA_METHOD" => "SMS" + case _ => "sweep-no-such-" + seg.toLowerCase.replace('_', '-') + } +} diff --git a/obp-api/src/test/scala/code/api/sweep/EndpointCatalogTest.scala b/obp-api/src/test/scala/code/api/sweep/EndpointCatalogTest.scala new file mode 100644 index 0000000000..aae0179f01 --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/EndpointCatalogTest.scala @@ -0,0 +1,87 @@ +package code.api.sweep + +import code.setup.ServerSetupWithTestData +import org.scalatest.Tag + +/** + * EndpointCatalog.isPlaceholder decides which ALL_CAPS URL segments the sweeps substitute with a + * concrete value before calling an endpoint. Getting this wrong in either direction breaks a + * sweep's verdict: substituting a real literal sends a value the endpoint does not recognise, and + * leaving a real placeholder unsubstituted does the same thing from the other side -- the sweep + * calls a well-formed request that was never going to work, and reports the resulting 4xx/5xx as + * the endpoint's own defect. + * + * This pins the second failure mode for the three segments EndpointCatalog's own comment named as + * a known, accepted gap: PAYMENT_SERVICE, PAYMENT_PRODUCT and SCA_METHOD are enumerated values a + * live endpoint validates inline, not literals -- Http4sBGv2PIS's payment-service branches guard + * on `Set("payments", "bulk-payments", "periodic-payments").contains(paymentService)`, and + * Http4s310's auth-context-updates branch guards on + * `List(StrongCustomerAuthentication.SMS, EMAIL).contains(scaMethod)`. Left as the literal + * strings "PAYMENT_SERVICE"/"SCA_METHOD", both guards fail and the sweep calls a URL that was + * never going to route anywhere -- exactly the class of self-inflicted failure EndpointCatalog's + * own docstring says the ID/_CODE/_NAME heuristic exists to avoid. + */ +class EndpointCatalogTest extends ServerSetupWithTestData { + + object EndpointCatalogPlaceholders extends Tag("EndpointCatalogPlaceholders") + + feature("EndpointCatalog substitutes every real path placeholder, not just the ones ending in ID/_CODE/_NAME") { + + scenario("PAYMENT_SERVICE is substituted with a value the endpoint's own guard would accept", + EndpointCatalogPlaceholders) { + // No ResourceDoc in the current EndpointCatalog carries a PAYMENT_SERVICE segment -- + // Berlin Group's route trees are not aggregated into Http4s700.allResourceDocs (only the + // OBP v1.2.1..v7.0.0 lineage is), so this exercises concretePath/isPlaceholder directly via + // .copy on a real doc rather than filtering the live catalog for one that is not there. + // That pins the behaviour EndpointCatalog must have the day Berlin Group docs do join the + // catalog, instead of waiting to notice the gap then. + val doc = EndpointCatalog.all.head.copy( + requestUrl = "/PAYMENT_SERVICE/PAYMENT_PRODUCT/PAYMENT_ID/status") + val path = EndpointCatalog.concretePath(doc) + withClue(s"($path) left PAYMENT_SERVICE unsubstituted -- Http4sBGv2PIS's payment-status " + + s"branch guards on Set(\"payments\",\"bulk-payments\",\"periodic-payments\")" + + s".contains(paymentService), so this literal fails it and the sweep would " + + s"misreport a working endpoint as broken: ") { + path should not include "PAYMENT_SERVICE" + } + } + + scenario("SCA_METHOD is substituted with a value the endpoint's own guard accepts", + EndpointCatalogPlaceholders) { + val docs = EndpointCatalog.all.filter(_.requestUrl.contains("SCA_METHOD")) + withClue("no ResourceDoc in the catalog carries a SCA_METHOD segment any more -- this " + + "test's premise no longer holds against the current catalog, update it: ") { + docs should not be empty + } + docs.foreach { doc => + val path = EndpointCatalog.concretePath(doc) + withClue(s"${doc.operationId} ($path) left SCA_METHOD unsubstituted -- the endpoint " + + s"only accepts SMS/EMAIL/IMPLICIT, so this literal fails validation and the " + + s"sweep misreports a working endpoint as broken: ") { + path should not include "SCA_METHOD" + } + } + } + + scenario("literal ALL_CAPS segments that are not placeholders stay untouched", + EndpointCatalogPlaceholders) { + // The fix must not turn every unrecognised ALL_CAPS segment into a placeholder -- only the + // three named above. CARDANO/MOBILE_WALLET/ETH_SEND_TRANSACTION are genuine literals this + // catalog must keep sending verbatim. + val literalSegments = List("CARDANO", "MOBILE_WALLET", "ETH_SEND_TRANSACTION") + val docs = EndpointCatalog.all.filter(doc => literalSegments.exists(doc.requestUrl.contains)) + withClue("no ResourceDoc in the catalog carries any of CARDANO/MOBILE_WALLET/" + + "ETH_SEND_TRANSACTION any more -- this test's premise no longer holds, update it: ") { + docs should not be empty + } + docs.foreach { doc => + val path = EndpointCatalog.concretePath(doc) + val literalInDoc = literalSegments.find(doc.requestUrl.contains).get + withClue(s"${doc.operationId} substituted over the literal $literalInDoc, which routes " + + s"nowhere -- these are not placeholders: ") { + path should include(literalInDoc) + } + } + } + } +} diff --git a/obp-api/src/test/scala/code/api/sweep/FailureSweepTest.scala b/obp-api/src/test/scala/code/api/sweep/FailureSweepTest.scala new file mode 100644 index 0000000000..b941e1082e --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/FailureSweepTest.scala @@ -0,0 +1,177 @@ +package code.api.sweep + +import cats.effect.IO +import cats.effect.unsafe.IORuntime +import code.api.util.APIUtil.ResourceDoc +import code.api.util.CustomJsonFormats +import code.api.util.http4s.Http4sApp +import code.setup.{DefaultUsers, ServerSetupWithTestData} +import fs2.Stream +import org.http4s.{Header, Headers, Method, Request, Uri} +import org.json4s.Extraction +import org.json4s.JValue +import org.json4s.JsonAST.JObject +import org.json4s.native.JsonMethods.{compact, render} +import org.scalatest.Tag +import org.typelevel.ci.CIString +import com.openbankproject.commons.util.JsonAliases.parse + +/** + * No endpoint answers a well-formed request with a 500. + * + * The auth sweep proves an endpoint refuses the wrong caller. This one proves it survives the + * right caller asking for something that is not there — which is the far more common shape of a + * production incident, and the one a migration is most likely to introduce. A connector that + * returns a slightly different empty value, a JSON codec that stops handling a null, a column + * read that no longer tolerates NULL: none of those show up as a wrong answer, they show up as + * a 500 on a request that used to work. + * + * That class of defect has a track record on this codebase. The first run of AuthSweepTest found + * `createTransactionRequestFreeForm` answering 500 to a nonexistent view id where it should have + * answered 403 or 404, and it found it by accident — the sweep was looking for something else. + * This suite looks for it on purpose, across every endpoint. + * + * ── What is sent ── + * + * A caller holding EVERY role, so nothing stops at the authorisation gate; path identifiers that + * are well-formed and do not exist; and, for verbs that take one, the endpoint's own + * `exampleRequestBody` serialised through the same json4s path the server uses to publish it. + * + * The example body is the right instrument here for the reason it is the wrong one for a success + * test: it is structurally valid and referentially meaningless. Its bank ids, currencies and + * entity references are illustrative, so an endpoint accepting it will almost always answer 400 + * or 404 — which is exactly the input that exercises the error paths, and exactly where an + * unhandled null or an over-narrow match arm turns into a 500. + * + * ── What is asserted ── + * + * 4xx fine, whatever the code. "Not found", "bad request", "not allowed" are all correct + * answers to a request for something that does not exist. + * 2xx fine. Some endpoints legitimately succeed with no arguments, or answer an empty list. + * 5xx a finding, always. + * + * Nothing here asserts WHICH 4xx. That would be a contract test, and the contract suite already + * owns it; asserting it twice, from a place with no baseline to compare against, would produce + * failures every time a message was reworded. + */ +object FailureSweepTest { + + /** + * The single definition of what this sweep covers. Exposed so SweepCoverageTest's drift check + * reads this directly instead of re-deriving its own copy of the same filter -- two copies of + * one expression are equal by construction and can never catch this sweep's own filtering + * changing independently of AuthSweepTest's. + */ + def scope: List[ResourceDoc] = EndpointCatalog.all.filter(EndpointCatalog.skipReason(_).isEmpty) +} + +class FailureSweepTest extends ServerSetupWithTestData with DefaultUsers with SweepFixtures { + + object FailureSweep extends Tag("FailureSweep") + + implicit val runtime: IORuntime = IORuntime.global + private lazy val app = Http4sApp.httpApp + + private def entities: Map[String, String] = + realBankId.map("BANK_ID" -> _).toList.toMap + + private def call(verb: String, path: String, headers: Map[String, String], body: String) + : (Int, JValue) = { + val method = Method.fromString(verb.toUpperCase).getOrElse(Method.GET) + val hdrs = if (body.nonEmpty) headers + ("Content-Type" -> "application/json") else headers + val req = Request[IO]( + method = method, + uri = Uri.unsafeFromString(path), + headers = Headers(hdrs.map { case (k, v) => Header.Raw(CIString(k), v) }.toList), + body = if (body.nonEmpty) Stream.emits(body.getBytes("UTF-8")).covary[IO] else Stream.empty + ) + val resp = app.run(req).unsafeRunSync() + val bodyStr = resp.bodyText.compile.string.unsafeRunSync() + val json = try { if (bodyStr.trim.isEmpty) JObject(Nil) else parse(bodyStr) } + catch { case _: Exception => JObject(Nil) } + (resp.status.code, json) + } + + /** + * The doc's own example body as JSON, or "" when it has none. + * + * Extraction.decompose under CustomJsonFormats is the same route Http4s uses to publish these + * objects, so what is sent is what the documentation shows a caller to send. + */ + private def exampleBody(doc: ResourceDoc): String = { + implicit val formats = CustomJsonFormats.formats + doc.exampleRequestBody match { + case null => "" + case body => + try compact(render(Extraction.decompose(body))) catch { case _: Exception => "" } + } + } + + /** + * Endpoints whose 5xx is the answer they are built to give. + * + * Exactly one so far, and it is not a real endpoint: Http4s700 registers + * `POST /obp/v7.0.0/test/rollback-check` inside `if (Props.testMode)` specifically to abort a + * transaction and prove the rollback happened, so a 500 is its pass condition. It exists only + * under `run.mode=test`, which is to say only where this sweep runs. + * + * Kept as a named map rather than removed from the catalog: SweepCoverageTest counts what the + * sweeps cover, and an endpoint quietly dropped from a list is the failure mode that whole + * test exists to prevent. Anything added here needs the same kind of reason. + */ + private val expected5xx: Map[String, String] = Map( + "OBPv7.0.0-testRollbackEndpoint" -> ("test-mode-only endpoint that deliberately throws to " + + "verify transaction rollback; its 500 IS the assertion (Http4s700, Props.testMode)"), + "OBPv7.0.0-createTestEmail" -> ("500 OBP-10056: refuses to send because portal_external_url " + + "is unset, which is true of any test rig. Environmental -- but a missing configuration is " + + "a 503, not a 500, so the status itself is logged in REGRESSION-GAPS rather than accepted " + + "as correct") + ) + + private lazy val inScope: List[ResourceDoc] = FailureSweepTest.scope + + private def check(doc: ResourceDoc, headers: Map[String, String], + ents: Map[String, String]): Option[String] = { + val path = EndpointCatalog.concretePath(doc, ents) + val body = if (doc.requestVerb.toUpperCase == "GET" || doc.requestVerb.toUpperCase == "DELETE") + "" else exampleBody(doc) + val (status, json) = call(doc.requestVerb, path, headers, body) + if (status >= 500 && !expected5xx.contains(doc.operationId)) { + implicit val formats = CustomJsonFormats.formats + val msg = (json \ "message").extractOpt[String].getOrElse("") + Some(s"${doc.operationId} ${doc.requestVerb} $path -> HTTP $status: $msg") + } else None + } + + private lazy val byVersion: Map[String, List[ResourceDoc]] = + inScope.groupBy(_.implementedInApiVersion.toString) + + feature("No endpoint answers a well-formed request with a server error") { + + byVersion.keys.toList.sorted.foreach { version => + scenario(s"$version -- a fully-entitled caller asking for something absent gets 4xx, never 5xx", + FailureSweep) { + setPropsValues("api_disabled_endpoints" -> "[]", "api_enabled_endpoints" -> "[]") + // Grant once per scenario, not once per class: beforeEach wipes the entitlement + // table, so a lazy val granted during the first scenario leaves every later one + // calling as an unentitled user -- which stops at 403 and never reaches the code + // that might crash. That is how the first run reported only two 5xx. + val headers = omniscientCaller + val ents = entities + val docs = byVersion(version) + + When(s"each of the ${docs.size} $version endpoints is called with valid credentials, " + + s"every role, a nonexistent id and its own example body") + val failures = docs.flatMap(check(_, headers, ents)) + + Then("none of them crashes") + withClue(s"${failures.size} of ${docs.size} $version endpoints answered 5xx. A request " + + s"for something that does not exist is an ordinary 404; a 500 means an " + + s"unhandled path, and it is the shape a migration introduces most often:\n" + + s"${failures.mkString("\n")}\n") { + failures shouldBe empty + } + } + } + } +} diff --git a/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala b/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala new file mode 100644 index 0000000000..66502a2349 --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/SuccessSweepTest.scala @@ -0,0 +1,177 @@ +package code.api.sweep + +import cats.effect.IO +import cats.effect.unsafe.IORuntime +import code.api.util.APIUtil.ResourceDoc +import code.api.util.CustomJsonFormats +import code.api.util.http4s.Http4sApp +import code.setup.{DefaultUsers, ServerSetupWithTestData} +import fs2.Stream +import org.http4s.{Header, Headers, Method, Request, Uri} +import org.json4s.JValue +import org.json4s.JsonAST.JObject +import org.scalatest.Tag +import org.typelevel.ci.CIString +import com.openbankproject.commons.util.JsonAliases.parse + +/** + * The endpoints that need nothing but a caller actually answer. + * + * The other two sweeps assert what happens when something is wrong: no credentials, no role, no + * such entity. Neither would notice an endpoint that had stopped returning data altogether — a + * GET that answers 404 for every caller passes the failure sweep, which only objects to 5xx. + * This one closes that: for the endpoints where a correct answer requires no setup at all, a + * fully-entitled caller must get one. + * + * ── Why only the no-path-variable endpoints ── + * + * Of the endpoints a caller can reach, roughly a third carry no ALL_CAPS placeholder in their + * URL — /banks, /users/current, /my/accounts, /management/metrics and so on. For those, "call it + * and expect an answer" is a complete test: there is no entity to create first, so a non-answer + * is the endpoint's own fault. + * + * The remaining two thirds are deliberately out of scope here. About half of them reference an + * identifier the fixtures do supply (BANK_ID, ACCOUNT_ID, VIEW_ID …) and the other half + * reference one nothing creates — CHAT_ROOM_ID, CUSTOMER_ID, CONSENT_ID and about twenty more. + * Both need per-cluster setup to be worth asserting, and a sweep that fabricated ids for them + * would be asserting 404-handling under a name that promises success. They are the next two + * waves, not this one. + * + * ── GET only ── + * + * The 116 no-variable POSTs are excluded because a generic success POST is not a thing: the + * example bodies are illustrative rather than referentially valid (which is exactly what makes + * them good failure-path input, see FailureSweepTest), and a POST that did succeed would leave + * a row behind that the next scenario's fixture reset may or may not clear. A write-path success + * sweep needs per-endpoint bodies and per-endpoint cleanup; that is Wave 3b's problem. + * + * ── What "an answer" means ── + * + * 2xx. Not a shape, not a field — the contract suite owns field-level assertions and has a + * baseline to compare against, which this does not. Asserting shape here would duplicate that + * work from a worse position and fail every time a message was reworded. + * + * Endpoints that legitimately cannot answer 2xx on a fixture database — because they need a + * connector this build does not have, or a feature the props disable — are listed in + * `expectedNon2xx` with the reason, and asserted to STILL not be 5xx. An endpoint that stops + * answering is a finding; an endpoint that was never going to answer here is a documented skip. + */ +class SuccessSweepTest extends ServerSetupWithTestData with DefaultUsers with SweepFixtures { + + object SuccessSweep extends Tag("SuccessSweep") + + implicit val runtime: IORuntime = IORuntime.global + private lazy val app = Http4sApp.httpApp + + /** + * Endpoints that answer non-2xx on a fixture database for a stated reason. + * + * Every entry is a claim that the non-answer is environmental, not a defect. They are still + * called, and still required not to crash — the skip is only from the 2xx assertion. Keeping + * them here rather than filtering them out of the catalog means SweepCoverageTest still counts + * them, and means each exemption has to be written down next to its reason. + */ + private val expectedNon2xx: Map[String, String] = Map( + // ── needs an external service this build does not run ── + "OBPv2.0.0-elasticSearchMetrics" -> "404: needs an Elasticsearch instance; none in the test rig", + "OBPv2.0.0-elasticSearchWarehouse" -> "404: needs an Elasticsearch instance; none in the test rig", + "OBPv2.2.0-getMessageDocs" -> ("400 OBP-30211: asks which connector's message docs to " + + "return; the fixture rig runs `mapped`, which publishes none"), + "OBPv3.1.0-getObpConnectorLoopback" -> "400 OBP-10010: not implemented by the mapped connector", + "OBPv6.0.0-getMessageDocsJsonSchema" -> "same as getMessageDocs -- no connector message docs to derive a schema from", + + // ── needs a certificate the test caller does not present ── + "OBPv5.1.0-mtlsClientCertificateInfo" -> ("400 OBP-20300: reports the caller's client " + + "certificate; these requests are driven in-process with no TLS peer"), + "OBPv4.0.0-verifyRequestSignResponse" -> ("401 OBP-20311: authenticates by JWS request " + + "signature rather than by session; the sweep signs nothing"), + + // ── the URL names an entity, in a segment shaped like a literal ── + // These carry ALL_CAPS segments that EndpointCatalog deliberately leaves verbatim + // (API_COLLECTION_NAME, WEBUI_PROP_NAME, SCHEME), so the server correctly reports that no + // such entity exists. Creating one first is Wave 3c's job, not this sweep's. + "OBPv4.0.0-getMyApiCollectionByName" -> "400 OBP-30079: no ApiCollection named API_COLLECTION_NAME", + "OBPv4.0.0-getMyApiCollectionEndpoints" -> "400 OBP-30079: no ApiCollection named API_COLLECTION_NAME", + "OBPv6.0.0-getWebUiProp" -> "400 OBP-08003: no WebUi prop named WEBUI_PROP_NAME", + "OBPv7.0.0-getRoutingScheme" -> "404 OBP-30514: no routing scheme named SCHEME" + ) + + private def get(path: String, headers: Map[String, String]): (Int, JValue) = { + val req = Request[IO]( + method = Method.GET, + uri = Uri.unsafeFromString(path), + headers = Headers(headers.map { case (k, v) => Header.Raw(CIString(k), v) }.toList), + body = Stream.empty + ) + val resp = app.run(req).unsafeRunSync() + val bodyStr = resp.bodyText.compile.string.unsafeRunSync() + val json = try { if (bodyStr.trim.isEmpty) JObject(Nil) else parse(bodyStr) } + catch { case _: Exception => JObject(Nil) } + (resp.status.code, json) + } + + /** + * No ALL_CAPS placeholder in the URL -- nothing to create before calling. + * + * EndpointCatalog.hasPlaceholder, not a local copy of the rule. This used to hold its own, + * and the two had already drifted: the catalog substitutes any segment ending in ID, _CODE or + * _NAME, this one looked for _ID and _CODE and had never learnt about _NAME. So + * `/signal/channels/CHANNEL_NAME/info` was a placeholder to the catalog -- which duly replaced + * it with a channel that does not exist -- and NOT a placeholder here, so this suite selected + * it as an endpoint that "needs nothing created first" and then failed it for answering 404. + * + * The first half of the old condition was worse than wrong, it was vacuous: + * `concretePath(doc) == concretePath(doc, Map.empty)` compares a default argument with the same + * value passed explicitly, so it is true for every doc and filtered nothing. + */ + private def hasNoPathVariable(doc: ResourceDoc): Boolean = !EndpointCatalog.hasPlaceholder(doc) + + private lazy val inScope: List[ResourceDoc] = + EndpointCatalog.all + .filter(EndpointCatalog.skipReason(_).isEmpty) + .filter(_.requestVerb.toUpperCase == "GET") + .filter(hasNoPathVariable) + + private lazy val byVersion: Map[String, List[ResourceDoc]] = + inScope.groupBy(_.implementedInApiVersion.toString) + + private def check(doc: ResourceDoc, headers: Map[String, String]): Option[String] = { + val path = EndpointCatalog.concretePath(doc) + val (status, json) = get(path, headers) + implicit val formats = CustomJsonFormats.formats + lazy val msg = (json \ "message").extractOpt[String].getOrElse("") + + if (status >= 500) + Some(s"${doc.operationId} GET $path -> HTTP $status (crash): $msg") + else if (status >= 200 && status < 300) + None + else expectedNon2xx.get(doc.operationId) match { + case Some(_) => None // documented environmental non-answer; not crashing is enough + case None => Some(s"${doc.operationId} GET $path -> HTTP $status: $msg") + } + } + + feature("Endpoints that require no setup answer a fully-entitled caller") { + + byVersion.keys.toList.sorted.foreach { version => + scenario(s"$version -- every no-argument GET returns data", SuccessSweep) { + setPropsValues("api_disabled_endpoints" -> "[]", "api_enabled_endpoints" -> "[]") + // Once per scenario -- beforeEach wipes the entitlement table, so a class-level + // lazy val would leave every scenario after the first calling without roles. + val headers = omniscientCaller + val docs = byVersion(version) + + When(s"each of the ${docs.size} $version GETs that need no path variable is called") + val failures = docs.flatMap(check(_, headers)) + + Then("each one answers") + withClue(s"${failures.size} of ${docs.size} $version no-argument GETs did not answer. " + + s"These need nothing created first, so a non-2xx is the endpoint's own. If one " + + s"of them cannot answer on a fixture database, add it to expectedNon2xx with " + + s"the reason rather than deleting the assertion:\n${failures.mkString("\n")}\n") { + failures shouldBe empty + } + } + } + } +} diff --git a/obp-api/src/test/scala/code/api/sweep/SweepCoverageDriftCheckTest.scala b/obp-api/src/test/scala/code/api/sweep/SweepCoverageDriftCheckTest.scala new file mode 100644 index 0000000000..8d0c37a4b1 --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/SweepCoverageDriftCheckTest.scala @@ -0,0 +1,72 @@ +package code.api.sweep + +import org.scalatest.{FlatSpec, Matchers} + +import java.io.File +import scala.io.Source + +/** + * Guards SweepCoverageTest's "the failure sweep covers the same set the auth sweep does" + * scenario against being vacuous. + * + * That scenario computes `authScope` and `failureScope` and asserts their symmetric difference + * is empty -- but if both are built by literally re-typing the same filter expression twice + * (`catalog.filter(EndpointCatalog.skipReason(_).isEmpty).map(_.operationId).toSet`, written out + * in SweepCoverageTest.scala rather than read from AuthSweepTest/FailureSweepTest themselves), + * the two values are equal BY CONSTRUCTION regardless of what those two sweeps actually iterate + * over. The assertion can then never fail, even after a real future divergence -- one sweep + * gains a filter of its own, and the endpoints that fall between them are covered by neither, + * silently, forever, because the "guard" was checking two copies of itself. + * + * The fix is for SweepCoverageTest to read AuthSweepTest.scope and FailureSweepTest.scope -- + * each sweep's own single definition of what it covers -- instead of re-deriving a copy. This + * test is a source scan rather than a runtime assertion because a value-equality check on the + * CURRENT catalog cannot distinguish "computed from the real source" from "coincidentally equal + * duplicate" -- both produce the identical Set today; the difference only matters for whether a + * FUTURE divergence gets caught, which a static duplicate can never do regardless of what the + * catalog looks like when the test runs. + */ +class SweepCoverageDriftCheckTest extends FlatSpec with Matchers { + + private def sourceOf(basename: String): String = { + val candidates = List( + new File(s"src/test/scala/code/api/sweep/$basename"), + new File(s"obp-api/src/test/scala/code/api/sweep/$basename") + ) + val file = candidates.find(_.isFile).getOrElse( + fail(s"Cannot locate $basename under either candidate path - this guard must not pass by " + + s"failing to look. Tried: ${candidates.mkString(", ")}")) + val source = Source.fromFile(file, "UTF-8") + try source.mkString finally source.close() + } + + private lazy val sweepCoverageSource = sourceOf("SweepCoverageTest.scala") + + "the drift-check scenario" should "read AuthSweepTest's own scope, not a re-derived copy" in { + withClue("SweepCoverageTest must reference AuthSweepTest.scope so a future change to " + + "AuthSweepTest's own filtering is automatically reflected here instead of silently " + + "diverging from a hand-copied duplicate: ") { + sweepCoverageSource should include("AuthSweepTest.scope") + } + } + + it should "read FailureSweepTest's own scope, not a re-derived copy" in { + withClue("SweepCoverageTest must reference FailureSweepTest.scope for the same reason: ") { + sweepCoverageSource should include("FailureSweepTest.scope") + } + } + + it should "not compute authScope/failureScope by writing the skipReason filter out twice" in { + val duplicateFilterPattern = + """catalog\.filter\(EndpointCatalog\.skipReason\(_\)\.isEmpty\)\.map\(_\.operationId\)\.toSet""".r + val occurrences = duplicateFilterPattern.findAllIn(sweepCoverageSource).length + withClue(s"found $occurrences occurrence(s) of the raw filter expression written directly " + + s"in SweepCoverageTest.scala. Two occurrences means authScope and failureScope are " + + s"each an independent copy of the same literal, equal by construction regardless of " + + s"what AuthSweepTest/FailureSweepTest actually cover -- the exact vacuousness this " + + s"guard exists to catch. Expected zero: the scopes should come from " + + s"AuthSweepTest.scope / FailureSweepTest.scope instead. ") { + occurrences shouldBe 0 + } + } +} diff --git a/obp-api/src/test/scala/code/api/sweep/SweepCoverageTest.scala b/obp-api/src/test/scala/code/api/sweep/SweepCoverageTest.scala new file mode 100644 index 0000000000..7033ab47ff --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/SweepCoverageTest.scala @@ -0,0 +1,153 @@ +package code.api.sweep + +import code.api.util.APIUtil.ResourceDoc +import code.setup.ServerSetupWithTestData +import org.scalatest.Tag + +/** + * The sweep is complete, and stays complete. + * + * A sweep driven off a registry has one failure mode that matters and that the sweep itself + * cannot see: an endpoint quietly leaving the swept set. Whether that happens because someone + * adds a skip, because a doc grows a tag, or because a filter is written slightly wrong, the + * result looks identical from outside — the sweep passes, and it passes over less than it did + * yesterday. This is the same shape as `run_tests_parallel.sh`'s "fewer than 2000 tests ran" + * floor and the contract suite's MIN_ATTEMPTED, moved inside the suite: prove the denominator + * before believing the numerator. + * + * The identity is exact, not a threshold: + * + * |catalog| == |swept| + |skipped| + * + * and every member of `skipped` carries one of the three reasons enumerated in EndpointCatalog. + * There is no fourth bucket. A sweep that wants to exclude something has to add a SkipReason + * with a written justification, in the file this test reads — which is the point. + */ +class SweepCoverageTest extends ServerSetupWithTestData { + + object SweepCoverage extends Tag("SweepCoverage") + + private lazy val catalog: List[ResourceDoc] = EndpointCatalog.all + + feature("The endpoint sweep covers every reachable endpoint, or says why not") { + + scenario("the catalog is non-empty and deduplicated by (url, verb)", SweepCoverage) { + Given("the aggregated v7.0.0 resource docs") + // A sweep over an empty catalog passes every assertion it makes. The floor is deliberately + // well below the ~870 observed on this branch: its job is to catch a catalog that failed to + // initialise, not to freeze a number that legitimately grows with every new endpoint. + withClue(s"catalog holds ${catalog.size} endpoints -- far below the expected several " + + s"hundred, which means the registry did not initialise and every sweep in this " + + s"package is asserting nothing. ") { + catalog.size should be > 400 + } + + Then("no (requestUrl, requestVerb) appears twice") + val duplicated = catalog + .groupBy(d => (d.requestUrl, d.requestVerb)) + .collect { case (key, docs) if docs.size > 1 => s"$key -> ${docs.map(_.operationId).mkString(", ")}" } + withClue(s"allResourceDocs is supposed to keep only the newest version of each " + + s"(url, verb); these survived twice:\n${duplicated.mkString("\n")}\n") { + duplicated shouldBe empty + } + } + + scenario("every endpoint is either swept or skipped for a stated reason", SweepCoverage) { + Given(s"the ${catalog.size} endpoints in the catalog") + val (skipped, swept) = catalog.partition(EndpointCatalog.skipReason(_).isDefined) + + Then("the two sets account for the catalog exactly, with nothing in between") + withClue(s"swept=${swept.size} skipped=${skipped.size} catalog=${catalog.size} -- these " + + s"must add up, or some endpoint is in a third bucket nobody is looking at. ") { + swept.size + skipped.size shouldBe catalog.size + } + + And("every skip names one of the enumerated reasons") + val unexplained = skipped.filter(EndpointCatalog.skipReason(_).isEmpty) + unexplained shouldBe empty + + And("the swept set is the large majority -- a skip list that has grown to swallow the " + + "catalog is a sweep that has stopped working") + withClue(s"only ${swept.size} of ${catalog.size} endpoints are swept; skips by reason: " + + s"${skipped.groupBy(EndpointCatalog.skipReason(_).get.why).view.mapValues(_.size).toMap}. ") { + swept.size should be > (catalog.size / 2) + } + } + + scenario("the auth classification is total -- every swept endpoint is public or protected", SweepCoverage) { + val swept = catalog.filter(EndpointCatalog.skipReason(_).isEmpty) + val protectedCount = swept.count(EndpointCatalog.needsAuthentication) + val publicCount = swept.size - protectedCount + + Then("the two classes partition the swept set") + protectedCount + publicCount shouldBe swept.size + + And("both classes are non-empty -- a classifier that answers the same for everything is " + + "not classifying") + withClue(s"protected=$protectedCount public=$publicCount. If either is zero the predicate " + + s"has stopped discriminating and both AuthSweepTest branches are vacuous. ") { + protectedCount should be > 0 + publicCount should be > 0 + } + } + + scenario("the failure sweep covers the same set the auth sweep does", SweepCoverage) { + // Read each sweep's OWN scope rather than re-deriving a copy here: today both are + // `EndpointCatalog.all.filter(EndpointCatalog.skipReason(_).isEmpty)`, so two independently + // hand-typed copies of that expression would be equal by construction and this scenario + // would pass even after a real future divergence -- one sweep grows a filter of its own, + // the endpoints that fall between the two are covered by neither, and nothing here would + // notice. Reading AuthSweepTest.scope / FailureSweepTest.scope means there is exactly one + // definition of each sweep's coverage, so a change to either is automatically reflected + // on both sides of this comparison. + val authScope = AuthSweepTest.scope.map(_.operationId).toSet + val failureScope = FailureSweepTest.scope.map(_.operationId).toSet + + val onlyAuth = authScope -- failureScope + val onlyFailure = failureScope -- authScope + withClue(s"endpoints swept for auth but not for crashes: ${onlyAuth.take(10).mkString(", ")}; " + + s"the reverse: ${onlyFailure.take(10).mkString(", ")}. ") { + onlyAuth shouldBe empty + onlyFailure shouldBe empty + } + } + + scenario("enough endpoints carry an example body for the failure sweep to exercise writers", + SweepCoverage) { + // FailureSweepTest sends exampleRequestBody to every non-GET endpoint. If almost none of + // them had one, the sweep would be a GET-only crash test wearing a broader name -- it + // would pass while every write path went unexercised. + val writers = catalog + .filter(EndpointCatalog.skipReason(_).isEmpty) + .filterNot(d => d.requestVerb.toUpperCase == "GET" || d.requestVerb.toUpperCase == "DELETE") + val withBody = writers.count(_.exampleRequestBody != null) + + withClue(s"$withBody of ${writers.size} write endpoints carry an exampleRequestBody. " + + s"Below half and the failure sweep is mostly not sending bodies at all. ") { + writers.size should be > 100 + withBody should be > (writers.size / 2) + } + } + + scenario("role-gated endpoints declare the errors their gate produces", SweepCoverage) { + val roleGated = catalog + .filter(EndpointCatalog.isRoleGated) + .filter(EndpointCatalog.roleSkipReason(_).isEmpty) + + Then("each one is also classified as needing authentication") + // The middleware derives auth from `errorResponseBodies contains AuthenticatedUserIsRequired + // OR roles.nonEmpty`, so a role-gated endpoint is authenticated by construction. If this + // ever fails, the predicate and the middleware have diverged. + val notAuthenticated = roleGated.filterNot(EndpointCatalog.needsAuthentication) + withClue(s"role-gated but not classified as needing authentication: " + + s"${notAuthenticated.map(_.operationId).mkString(", ")}. ") { + notAuthenticated shouldBe empty + } + + And("there are enough of them for the 403 sweep to be meaningful") + withClue(s"only ${roleGated.size} role-gated endpoints are in scope for the 403 assertion. ") { + roleGated.size should be > 100 + } + } + } +} diff --git a/obp-api/src/test/scala/code/api/sweep/SweepFixtures.scala b/obp-api/src/test/scala/code/api/sweep/SweepFixtures.scala new file mode 100644 index 0000000000..c029f0bf56 --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/SweepFixtures.scala @@ -0,0 +1,41 @@ +package code.api.sweep + +import code.api.util.ApiRole +import code.entitlement.Entitlement +import code.setup.DefaultUsers + +/** + * Shared setup for sweeps that call the API with a fully-entitled caller. + * + * FailureSweepTest and SuccessSweepTest each grew an identical "grant every role, then build a + * DirectLogin header" construction, and AuthSweepTest, FailureSweepTest and SuccessSweepTest each + * looked up the fixture bank independently. One shared definition here, called from all three, + * means a future change to either only has one site to update. + */ +trait SweepFixtures { self: DefaultUsers => + + /** The first sandbox bank the fixtures created, if any. */ + def realBankId: Option[String] = + code.bankconnectors.LocalMappedConnector.getBanksLegacy(None) + .map(_._1).getOrElse(Nil).headOption.map(_.bankId.value) + + /** + * A caller holding every role in the system. + * + * Granted directly through the Entitlement provider rather than over the API -- the same thing + * 161 existing test files do -- because the goal is to get PAST authorisation, not to test it. + */ + def omniscientCaller: Map[String, String] = { + ApiRole.availableRoles.foreach { role => + // Bank-scoped roles need a bank; system-wide ones must be granted with an empty bankId. + // valueOf throws on a name it does not recognise, and availableRoles includes dynamic + // roles whose backing entity may not exist in this database -- a grant that cannot be + // made is not a reason to abandon the other several hundred. + try { + val bankId = if (ApiRole.valueOf(role).requiresBankId) realBankId.getOrElse("") else "" + Entitlement.entitlement.vend.addEntitlement(bankId, resourceUser1.userId, role) + } catch { case _: Exception => () } + } + Map("DirectLogin" -> s"token=${token1.value}") + } +} diff --git a/obp-api/src/test/scala/code/api/sweep/SweepFixturesDuplicationTest.scala b/obp-api/src/test/scala/code/api/sweep/SweepFixturesDuplicationTest.scala new file mode 100644 index 0000000000..ae947a1ec2 --- /dev/null +++ b/obp-api/src/test/scala/code/api/sweep/SweepFixturesDuplicationTest.scala @@ -0,0 +1,69 @@ +package code.api.sweep + +import org.scalatest.{FlatSpec, Matchers} + +import java.io.File +import scala.io.Source + +/** + * Guards against the "grant every role" and "find a real bank id" constructions drifting back + * into byte-for-byte duplicates across the sweep test files. + * + * FailureSweepTest.omniscientUser and SuccessSweepTest.entitledCaller started as identical + * bodies -- grant every ApiRole to resourceUser1, then build a DirectLogin header -- and + * AuthSweepTest, FailureSweepTest and SuccessSweepTest each looked up + * LocalMappedConnector.getBanksLegacy(None) independently. None of it lived in EndpointCatalog, + * the module this package already treats as the one place shared sweep logic belongs (see + * AuthSweepTest.scope / FailureSweepTest.scope and SweepCoverageDriftCheckTest, which exists for + * exactly this reason on a different pair of definitions). + * + * A source scan, not a runtime assertion, for the same reason SweepCoverageDriftCheckTest is one: + * a value-equality check on today's fixtures cannot distinguish "computed from one shared + * definition" from "two copies that happen to still agree" -- both look identical today, and the + * difference only matters for whether a FUTURE divergence gets caught. + */ +class SweepFixturesDuplicationTest extends FlatSpec with Matchers { + + private def sourceOf(basename: String): String = { + val candidates = List( + new File(s"src/test/scala/code/api/sweep/$basename"), + new File(s"obp-api/src/test/scala/code/api/sweep/$basename") + ) + val file = candidates.find(_.isFile).getOrElse( + fail(s"Cannot locate $basename under either candidate path - this guard must not pass by " + + s"failing to look. Tried: ${candidates.mkString(", ")}")) + val source = Source.fromFile(file, "UTF-8") + try source.mkString finally source.close() + } + + "the 'grant every role, then build a DirectLogin header' construction" should + "appear once, in a shared fixture, not once per sweep file" in { + val pattern = """ApiRole\.availableRoles\.foreach""".r + val occurrences = List("SweepFixtures.scala", "FailureSweepTest.scala", "SuccessSweepTest.scala") + .map(f => f -> pattern.findAllIn(sourceOf(f)).length) + val total = occurrences.map(_._2).sum + + withClue(s"occurrences per file: ${occurrences.mkString(", ")}. Two independent copies of " + + s"the same 'grant every role' construction means a future change to how the " + + s"omniscient test caller is built (a new role category, a different bank-selection " + + s"rule) has to be applied by hand in both files, with nothing enforcing they stay " + + s"in sync. Expected exactly one, in a fixture both files call. ") { + total shouldBe 1 + } + } + + "the LocalMappedConnector.getBanksLegacy(None) bank lookup" should + "appear once, in a shared fixture, not once per sweep file" in { + val pattern = """LocalMappedConnector\.getBanksLegacy\(None\)""".r + val occurrences = List("SweepFixtures.scala", "AuthSweepTest.scala", "FailureSweepTest.scala", "SuccessSweepTest.scala") + .map(f => f -> pattern.findAllIn(sourceOf(f)).length) + val total = occurrences.map(_._2).sum + + withClue(s"occurrences per file: ${occurrences.mkString(", ")}. Three independent copies of " + + s"the same bank lookup means a future change to how the fixture bank is found has " + + s"to be applied by hand in three places, with nothing enforcing they stay in sync. " + + s"Expected exactly one, in a fixture all three files call. ") { + total shouldBe 1 + } + } +} diff --git a/obp-api/src/test/scala/code/api/util/AgentDelegationTest.scala b/obp-api/src/test/scala/code/api/util/AgentDelegationTest.scala index 3171733e8b..571480a9a7 100644 --- a/obp-api/src/test/scala/code/api/util/AgentDelegationTest.scala +++ b/obp-api/src/test/scala/code/api/util/AgentDelegationTest.scala @@ -14,7 +14,7 @@ import org.scalatest.Tag * - LiftUsers.createResourceUser field assignments — pins the 2021 copy-paste bug where * the createdByUserInvitationId None branch wiped CreatedByConsentId (the consent → * agent linkage every delegation query joins through). - * - CallContext.effectiveHumanUserId — resolve-up from the authenticated caller (human + * - CallContext.accountableUserId — resolve-up from the authenticated caller (human * or consent-minted agent) to the human the request is really about, including the * branch an HTTP test cannot reach (the agent as the caller). */ @@ -71,23 +71,23 @@ class AgentDelegationTest extends ServerSetup { } } - feature("CallContext.effectiveHumanUserId resolves the caller to the human the request is about") { + feature("CallContext.accountableUserId resolves the caller to the human the request is about") { scenario("a plain human resolves to themselves", AgentDelegationTag) { val human = createUser() - CallContext(user = Full(human)).effectiveHumanUserId shouldBe human.userId + CallContext(user = Full(human)).accountableUserId shouldBe human.userId } scenario("a consent-minted agent resolves to the granting human", AgentDelegationTag) { val human = createUser() val consent = MappedConsent.create.mUserId(human.userId).saveMe() val agent = createUser(createdByConsentId = Some(consent.consentId)) - CallContext(user = Full(agent)).effectiveHumanUserId shouldBe human.userId + CallContext(user = Full(agent)).accountableUserId shouldBe human.userId } scenario("an agent with a dangling consent id falls back to itself (fails closed)", AgentDelegationTag) { val agent = createUser(createdByConsentId = Some(generateUUID())) - CallContext(user = Full(agent)).effectiveHumanUserId shouldBe agent.userId + CallContext(user = Full(agent)).accountableUserId shouldBe agent.userId } scenario("a populated consenter box wins over the DB chain", AgentDelegationTag) { @@ -96,7 +96,7 @@ class AgentDelegationTest extends ServerSetup { val agent = createUser(createdByConsentId = Some(consent.consentId)) val consenterHuman = createUser() CallContext(user = Full(agent), consenter = Full(consenterHuman)) - .effectiveHumanUserId shouldBe consenterHuman.userId + .accountableUserId shouldBe consenterHuman.userId } scenario("onBehalfOfUser wins over consenter", AgentDelegationTag) { @@ -107,7 +107,7 @@ class AgentDelegationTest extends ServerSetup { user = Full(agent), consenter = Full(consenterHuman), onBehalfOfUser = Full(explicitHuman) - ).effectiveHumanUserId shouldBe explicitHuman.userId + ).accountableUserId shouldBe explicitHuman.userId } } } diff --git a/obp-api/src/test/scala/code/api/util/DynamicUtilJavaHttp4sEndpointTest.scala b/obp-api/src/test/scala/code/api/util/DynamicUtilJavaHttp4sEndpointTest.scala new file mode 100644 index 0000000000..ab984a317b --- /dev/null +++ b/obp-api/src/test/scala/code/api/util/DynamicUtilJavaHttp4sEndpointTest.scala @@ -0,0 +1,109 @@ +package code.api.util + +import cats.effect.IO +import cats.effect.unsafe.implicits.global +import org.http4s.{Method, Request, Uri} +import org.json4s.native.JsonMethods.parse +import org.scalatest.{FeatureSpec, GivenWhenThen, Matchers} + +/** + * Focused unit test for DynamicUtil.createJavaHttp4sEndpoint, isolated from the full + * register -> role-check -> HTTP-dispatch round trip (see DynamicResourceDocJavaTest for that). + * Exercises the adapter directly: compiled Java Supplier> -> + * Http4sEndpointIO.apply(Request[IO]) -> CallContext => IO[Response[IO]]. + */ +class DynamicUtilJavaHttp4sEndpointTest extends FeatureSpec with Matchers with GivenWhenThen { + + private val echoMethodBody = + """package code.api.util.dynamic; + | + |import code.api.util.CallContext; + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaHttp4sEndpointUnitTest implements Supplier> { + | private Object apply(Object[] args) { + | String rawBody = (String) args[0]; + | @SuppressWarnings("unchecked") + | Map pathParams = (Map) args[1]; + | CallContext cc = (CallContext) args[2]; + | + | Map response = new LinkedHashMap<>(); + | response.put("echoed_body", rawBody); + | response.put("path_param_count", pathParams.size()); + | response.put("correlation_id", cc.correlationId()); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + feature("DynamicUtil.createJavaHttp4sEndpoint compiles a Java method_body into a native Http4sEndpointIO") { + + scenario("the compiled endpoint reads args(0)/args(1)/args(2) and serves 200 JSON") { + Given("a Java method_body compiled via createJavaHttp4sEndpoint") + val endpoint = DynamicUtil.createJavaHttp4sEndpoint(echoMethodBody).openOrThrowException("compilation failed") + + When("the compiled endpoint handles a request carrying a body, no path params, and a CallContext") + val req = Request[IO](method = Method.POST, uri = Uri.unsafeFromString("/test")) + val cc = CallContext(httpBody = Some("""{"hello":"world"}"""), correlationId = "test-correlation-id") + val resp = endpoint.apply(req)(cc).unsafeRunSync() + + Then("the response is 200 and echoes the body, the (empty) path params, and the CallContext's correlationId") + resp.status.code should equal(200) + val bodyString = resp.body.through(fs2.text.utf8.decode).compile.string.unsafeRunSync() + val json = parse(bodyString) + (json \ "echoed_body").values should equal("""{"hello":"world"}""") + (json \ "path_param_count").values should equal(BigInt(0)) + (json \ "correlation_id").values should equal("test-correlation-id") + } + + scenario("a Java compile error is reported as a Box Failure, not a thrown exception") { + Given("a method_body that is not valid Java") + val badMethodBody = "this is not valid java at all" + + When("we try to compile it") + val result = DynamicUtil.createJavaHttp4sEndpoint(badMethodBody) + + Then("compilation fails gracefully") + result.isDefined should equal(false) + } + + scenario("a Java method_body that throws at runtime is recovered as a 500, not an uncaught exception") { + Given("a Java method_body whose apply() throws") + val throwingMethodBody = + """package code.api.util.dynamic; + | + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaHttp4sEndpointThrowingTest implements Supplier> { + | private Object apply(Object[] args) { + | throw new RuntimeException("boom"); + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + val endpoint = DynamicUtil.createJavaHttp4sEndpoint(throwingMethodBody).openOrThrowException("compilation failed") + + When("the compiled endpoint is invoked") + val req = Request[IO](method = Method.POST, uri = Uri.unsafeFromString("/test")) + val resp = endpoint.apply(req)(CallContext()).unsafeRunSync() + + Then("the response is 500 rather than the IO failing") + resp.status.code should equal(500) + val bodyString = resp.body.through(fs2.text.utf8.decode).compile.string.unsafeRunSync() + bodyString should include("boom") + } + } +} diff --git a/obp-api/src/test/scala/code/api/util/EndpointMappingCacheInvalidationTest.scala b/obp-api/src/test/scala/code/api/util/EndpointMappingCacheInvalidationTest.scala new file mode 100644 index 0000000000..e801297e0e --- /dev/null +++ b/obp-api/src/test/scala/code/api/util/EndpointMappingCacheInvalidationTest.scala @@ -0,0 +1,59 @@ +package code.api.util + +import code.api.JedisMethod +import code.api.cache.Redis +import code.setup.RedisTestTarget +import org.scalatest.{FlatSpec, Matchers} + +/** + * `NewStyle.function.invalidateEndpointMappingCache` guards against the write-then-invalidate + * race that comes with the cache key fix in `getEndpointMappingsCached`: while CallContext was + * part of the memoize key nothing could ever hit, so a stale entry was unreachable by + * construction. Now that the cache genuinely hits, a reader that fetched the pre-write value a + * moment earlier can still complete its own cache write AFTER the immediate delete finishes, + * silently reintroducing the stale entry for the rest of `endpointMapping.cache.ttl.seconds` -- + * nothing else would clear it before the next write. + * + * Racing two real threads against real DB/Redis latency would make this test flaky by + * construction (the window it is trying to hit is exactly the thing that is nondeterministic). + * Planting a key that matches the invalidation glob directly stands in for the straggler write + * instead -- the same deterministic-simulation technique IdempotencyMiddlewareTest uses for its + * own concurrency scenarios -- and this asserts the mechanism that is supposed to catch it: a + * second, delayed delete. + */ +class EndpointMappingCacheInvalidationTest extends FlatSpec with Matchers { + + private def redis(): Unit = + RedisTestTarget.requireReachable(Redis.isRedisReady, "the endpoint-mapping cache invalidation race guard") + + "invalidateEndpointMappingCache" should "clear a straggler entry that lands after the immediate delete" in { + redis() + // Any key matching the same glob the real memoized entry would (*getEndpointMappings*) + // stands in for the straggler -- the exact scalacache-derived key shape is not what this + // guards, only that a second sweep eventually clears whatever landed in the gap. + val stragglerKey = "test_ns:code.api.util.NewStyle.function.getEndpointMappingsCached(Some(straggler))()" + Redis.use(JedisMethod.SET, stragglerKey, None, Some("[]")) + withClue("test setup failed to plant the straggler key: ") { + Redis.use(JedisMethod.GET, stragglerKey, None, None) shouldBe Some("[]") + } + + NewStyle.function.invalidateEndpointMappingCache() + + withClue("immediately after the call the straggler should already be gone once, but that " + + "alone does not prove there is a SECOND delete -- see below") { + Redis.use(JedisMethod.GET, stragglerKey, None, None) shouldBe None + } + + // Simulate the race: the straggler write lands in the gap between the immediate delete and + // the scheduled one. + Redis.use(JedisMethod.SET, stragglerKey, None, Some("[]")) + + Thread.sleep(NewStyle.function.endpointMappingCacheInvalidationDelay.toMillis + 300) + + withClue("the delayed second invalidation must still clear a straggler that landed after " + + "the first delete, or a concurrent read racing the write can leave a stale " + + "endpoint-mapping list cached for the rest of the TTL: ") { + Redis.use(JedisMethod.GET, stragglerKey, None, None) shouldBe None + } + } +} diff --git a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala index 520d04a223..3816545da8 100644 --- a/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala +++ b/obp-api/src/test/scala/code/api/util/ResourceDocRegistryParityTest.scala @@ -1,21 +1,24 @@ package code.api.util +import code.api.berlin.group.ConstantsBG +import code.api.berlin.group.v1_3.Http4sBGv13Alias import code.setup.ServerSetup +import com.openbankproject.commons.util.{ApiStandards, ApiVersion, ScannedApiVersion} import org.scalatest.Tag /** - * Guards the invariant that APIUtil.getAllResourceDocs — the global operation-id - * registry used wherever an operation id must be resolved (api-collection endpoint - * validation, top-apis operation-id lookups, ...) — contains every per-standard - * resource-doc surface the resource-docs dispatcher can serve to API Explorer. + * Guards the invariant that APIUtil.getAllResourceDocs — the global operation-id registry used + * wherever an operation id must be resolved (api-collection endpoint validation, top-apis + * operation-id lookups, ...) — contains every per-standard resource-doc surface the resource-docs + * dispatcher can serve to API Explorer. * - * These are two parallel registries (ResourceDocsAPIMethods dispatches per - * standard/version; getAllResourceDocs aggregates them all), and they have drifted - * twice: Berlin Group v2 was served by the dispatcher but missing from the global - * registry (so BGv2-getAccountDetails could not be added to an API collection), - * and the global registry was based on the v6 aggregation, excluding v7-only - * operation ids. When you add a NEW API standard, register its docs in BOTH - * places — and add its surface to this list. + * Both sides are now derived from the single ResourceDocRegistry.registry map, so this class of + * drift (which happened three times by hand: Berlin Group v2, v7-only operation ids, and the + * Berlin Group v1.3 alias) is structurally impossible going forward — see ResourceDocRegistry's + * doc comment. This test's job is narrower than it used to be: it iterates the registry itself + * (rather than a hand-typed list of standards) so it stays correct as standards are added or + * removed without needing an edit here, and it catches an accidental regression back to two + * independently hand-maintained registries. */ class ResourceDocRegistryParityTest extends ServerSetup { @@ -24,19 +27,45 @@ class ResourceDocRegistryParityTest extends ServerSetup { private lazy val allOperationIds: Set[String] = APIUtil.getAllResourceDocs.map(_.operationId).toSet - private lazy val surfaces: List[(String, Seq[String])] = List( - ("OBP standard (v7 aggregation)", code.api.v7_0_0.Http4s700.allResourceDocs.map(_.operationId).toSeq), - ("Berlin Group v1.3", code.api.berlin.group.v1_3.Http4sBGv13.resourceDocs.map(_.operationId).toSeq), - ("Berlin Group v2", code.api.berlin.group.v2.Http4sBGv2.resourceDocs.map(_.operationId).toSeq), - ("UK Open Banking 2.0.0", code.api.UKOpenBanking.v2_0_0.OBP_UKOpenBanking_200.allResourceDocs.map(_.operationId).toSeq), - ("UK Open Banking 3.1.0", code.api.UKOpenBanking.v3_1_0.OBP_UKOpenBanking_310.allResourceDocs.map(_.operationId).toSeq), - ("UK Open Banking 4.0.1", code.api.UKOpenBanking.v4_0_1.OBP_UKOpenBanking_401.allResourceDocs.map(_.operationId).toSeq) - ) + // The Berlin Group v1.3 alias is the one surface a deployment can switch off: + // berlin_group_v1_3_alias_path is unset by default and is supplied for test runs by + // test.default.props and by the two CI workflows. test.default.props is gitignored + // (.gitignore:21), so a fresh clone, a colleague's checkout or an IDE ScalaTest run may not have + // it -- the pin below cancels there instead of failing with a message that gives no hint a prop + // is missing. The per-surface loop needs no such guard: an unconfigured alias reports the + // unaddressable ScannedApiVersion("", "", ""), which ScannedApis now drops, so it is not a + // surface at all rather than an empty one. + private lazy val aliasIsConfigured: Boolean = Http4sBGv13Alias.resourceDocs.nonEmpty + private val aliasNotConfigured = + "berlin_group_v1_3_alias_path is not set, so the Berlin Group v1.3 alias contributes no docs" + + private def label(version: ApiVersion): String = version match { + case sv: ScannedApiVersion => sv.fullyQualifiedVersion + case other => other.toString + } + + // Scoped to ResourceDocRegistry.unionVersions -- the current OBP surface plus every non-OBP + // standard. The superseded OBP aggregations (v6.0.0 and older) and the two dynamic arms are + // deliberately out of the union; see ResourceDocRegistry.obpUnionVersion for why, and for the + // accepted consequence that an operation id living only in a superseded aggregation stays + // unresolvable. + private lazy val surfaces: List[(String, Seq[String])] = + ResourceDocRegistry.unionVersions.toList + .map(version => (label(version), ResourceDocRegistry.docsFor(version).map(_.operationId))) + + feature("getAllResourceDocs contains every per-standard resource-doc surface the union covers") { + scenario("the registry itself is non-empty", RegistryParityTag) { + surfaces should not be empty + } - feature("getAllResourceDocs contains every per-standard resource-doc surface") { surfaces.foreach { case (label, operationIds) => scenario(s"$label operation ids are all resolvable globally", RegistryParityTag) { - operationIds should not be empty + // Non-empty matters as much as membership: an empty surface is trivially a subset of the + // union, so without this a standard whose docs silently stop being registered (the very + // failure mode this test exists for) would pass unnoticed. + withClue(s"$label contributed no operation ids at all -- did its docs stop being registered? ") { + operationIds should not be empty + } val missing = operationIds.filterNot(allOperationIds.contains) withClue(s"$label operation ids missing from getAllResourceDocs: ${missing.take(10).mkString(", ")} ") { missing shouldBe empty @@ -44,8 +73,113 @@ class ResourceDocRegistryParityTest extends ServerSetup { } } + // Guards the one hand-maintained knob left in the registry: if a v8.0.0 aggregation is added + // without moving obpUnionVersion, the union would keep serving the v7 surface and every + // v8-only operation id would silently be unresolvable -- the exact bug this PR started from. + scenario("obpUnionVersion is the newest OBP-standard version in the registry", RegistryParityTag) { + val obpVersions = ResourceDocRegistry.registry.keys.toList.collect { + case sv: ScannedApiVersion + if sv.apiStandard == ApiStandards.obp.toString && + sv != ApiVersion.`dynamic-endpoint` && sv != ApiVersion.`dynamic-entity` => sv + } + obpVersions should not be empty + + // Rank by position in ApiVersionUtils.versions, which lists the OBP versions oldest-first. + // indexOf returns -1 for anything absent from that (also hand-maintained) list, and a -1 + // would lose every maxBy comparison -- so a v8.0.0 added to the registry but not to + // ApiVersionUtils.versions would leave v7 as the maximum and let this scenario pass, in + // exactly the two-places-to-edit case it exists to catch. Establish coverage first. + val unranked = obpVersions.filter(ApiVersionUtils.versions.indexOf(_) < 0) + withClue(s"OBP versions in the registry but missing from ApiVersionUtils.versions: " + + s"${unranked.map(_.fullyQualifiedVersion).mkString(", ")} -- add them there so they can be " + + s"ranked, otherwise this guard cannot see them ") { + unranked shouldBe empty + } + + val newest = obpVersions.maxBy(ApiVersionUtils.versions.indexOf(_)) + withClue(s"registry holds OBP versions ${obpVersions.map(_.fullyQualifiedVersion).mkString(", ")} " + + s"but obpUnionVersion is ${ResourceDocRegistry.obpUnionVersion} ") { + newest shouldBe ResourceDocRegistry.obpUnionVersion + } + } + + // Berlin Group and UK Open Banking both publish getBalances, getAccountList and + // getAccountBalances. Http4s600's top-apis/popular-apis and JSONFactory6.0.0's metrics resolve + // a partialFunctionName with `.toMap`, which keeps the LAST matching entry, so the registry's + // iteration order decides the operation_id those endpoints report. The hand-written union that + // preceded this registry listed UK before BG, giving Berlin Group the three names; sorting the + // scanned standards alphabetically silently handed them to UK Open Banking instead. This pins + // the resolved values so the precedence cannot drift again unnoticed. + scenario("Berlin Group keeps the partialFunctionNames it shares with UK Open Banking", RegistryParityTag) { + val resolved = APIUtil.getAllResourceDocs + .map(doc => doc.partialFunctionName -> doc.operationId).toMap + resolved.get("getBalances") shouldBe Some("BGv1.3-getBalances") + resolved.get("getAccountList") shouldBe Some("BGv2-getAccountList") + resolved.get("getAccountBalances") shouldBe Some("BGv2-getAccountBalances") + } + + // The Berlin Group v1.3 alias only re-publishes the canonical BG v1.3 docs, so it must never + // win a partialFunctionName away from the standard it copied. Its apiStandard is the first + // segment of berlin_group_v1_3_alias_path, so a deployment can point it at a name an existing + // standard already uses ("BG/v9"); ranking by that string alone put the alias alongside Berlin + // Group and, sorting after "v2", ahead of it. Ranking is by identity instead, and the synthetic + // alias below exercises the colliding configuration without needing a JVM under that prop. + scenario("a derived alias never outranks the standard it re-publishes", RegistryParityTag) { + val syntheticAlias = ScannedApiVersion("BG", "BG", "v9") + val rankOf = ResourceDocRegistry.sortKey(syntheticAlias) _ + withClue("the alias must sort before Berlin Group, i.e. lose the `.toMap` last-wins race ") { + rankOf(syntheticAlias) should be < rankOf(ConstantsBG.berlinGroupVersion2) + rankOf(syntheticAlias) should be < rankOf(ConstantsBG.berlinGroupVersion1) + } + withClue("the alias must also sort before UK Open Banking ") { + rankOf(syntheticAlias) should be < rankOf(ApiVersion.ukOpenBankingV401) + } + withClue("UK must still sort before Berlin Group, so BG keeps the names they share ") { + rankOf(ApiVersion.ukOpenBankingV401) should be < rankOf(ConstantsBG.berlinGroupVersion2) + } + } + + // An unconfigured configuration-gated standard reports ScannedApiVersion("", "", ""), whose + // fullyQualifiedVersion is "" as well. While ScannedApis kept it, ApiVersionUtils.valueOf("") + // resolved successfully and GET /obp/v7.0.0/resource-docs//obp answered 200 with an empty + // document list instead of the 400 every other unknown version string gets. + scenario("an unaddressable empty version is not a registered API version", RegistryParityTag) { + ScannedApis.versionMapScannedApis.keys.foreach { version => + withClue(s"$version was registered despite addressing nothing ") { + (version.urlPrefix.trim + version.apiStandard.trim + version.apiShortVersion.trim) should not be empty + } + } + ApiVersionUtils.versions.map(_.fullyQualifiedVersion) should not contain "" + an[IllegalArgumentException] should be thrownBy ApiVersionUtils.valueOf("") + } + + // The three named pins below are the three historical drift instances. They are NOT redundant + // with the loop above: both sides of that loop are now derived from ResourceDocRegistry, so its + // membership half holds by construction and cannot fail. What the loop still catches is a + // surface going empty; what these pins still catch is a specific operation id disappearing. + scenario("the operation id from the sandbox bug report resolves", RegistryParityTag) { allOperationIds should contain("BGv2-getAccountDetails") } + + // The alias's operation-id prefix is derived from the configured path (0.6/v1 in the test props + // yields BGv1-...), so the expected id is read back from the alias's own docs rather than + // hard-coded -- a deployment that configures a different path would otherwise fail here for no + // real reason. + scenario("the operation id from the Berlin Group v1.3 alias resolves", RegistryParityTag) { + if (!aliasIsConfigured) cancel(aliasNotConfigured) + val aliasOperationId = Http4sBGv13Alias.resourceDocs + .find(_.partialFunctionName == "getPaymentInitiationStatus").map(_.operationId) + withClue("the alias is configured but publishes no getPaymentInitiationStatus doc ") { + aliasOperationId shouldBe defined + } + allOperationIds should contain(aliasOperationId.get) + } + + // The union used to be built from the v6.0.0 aggregation, so v7-only operation ids were + // absent from it. getMyMetrics exists only in v7.0.0, so it pins the v7 base specifically. + scenario("a v7-only operation id resolves", RegistryParityTag) { + allOperationIds should contain("OBPv7.0.0-getMyMetrics") + } } } diff --git a/obp-api/src/test/scala/code/api/util/http4s/IdempotencyMiddlewareTest.scala b/obp-api/src/test/scala/code/api/util/http4s/IdempotencyMiddlewareTest.scala new file mode 100644 index 0000000000..bdc09eeab7 Binary files /dev/null and b/obp-api/src/test/scala/code/api/util/http4s/IdempotencyMiddlewareTest.scala differ diff --git a/obp-api/src/test/scala/code/api/v2_1_0/SandboxDataLoadingTest.scala b/obp-api/src/test/scala/code/api/v2_1_0/SandboxDataLoadingTest.scala index 646fdb18de..ae78699734 100644 --- a/obp-api/src/test/scala/code/api/v2_1_0/SandboxDataLoadingTest.scala +++ b/obp-api/src/test/scala/code/api/v2_1_0/SandboxDataLoadingTest.scala @@ -1082,6 +1082,53 @@ class SandboxDataLoadingTest extends FlatSpec with SendServerRequests with Match Connector.connector.vend.getBankAccountLegacy(BankId(acc1.bank), AccountId(acc2.id), None).isDefined should equal(true) } + it should "not allow two accounts at DIFFERENT banks to share an IBAN either" in { + // The global-uniqueness half of the rule, which had no test at all. + // + // An IBAN is globally unique by ISO 13616 -- the bank identifier is encoded INSIDE the + // string, so two banks cannot legitimately hold the same one. OBP does not merely assume + // that; it depends on it. LocalMappedConnector.getBankAccountByRoutingLegacy, called with + // no bankId, refuses outright when a routing address matches more than one account: + // + // if (routing.size > 1) { // Routing MUST be unique + // Failure(s"$AccountRoutingNotUnique (scheme: $scheme, address: $address)") + // + // and that is the lookup PAYMENT DESTINATIONS resolve through -- BulkPaymentHandler and + // three v7.0.0 transaction paths all call it with bankId = None. So letting a duplicate in + // at import time does not create a working account: it creates one that any global-routing + // payment then fails on, far from the import that caused it. + // + // Rejecting at import is therefore the correct behaviour, and this test exists so nobody + // "fixes" the duplicate check by scoping it per bank to make a broken fixture load. + val users = standardUsers + val banks = standardBanks + + def getResponse(accountJsons : List[JValue]) = { + BankAccountRouting.bulkDelete_!!() + val json = createImportJson(banks.map(Extraction.decompose), users.map(Extraction.decompose), accountJsons, Nil, Nil, Nil, Nil, Nil) + postImportJson(json) + } + + val accAtBank1 = account1AtBank1 + val accAtBank2 = account1AtBank2 + + val bank1Json = Extraction.decompose(accAtBank1) + // Same IBAN, different bank. Nothing else changed. + val bank2SameIbanJson = replaceField(Extraction.decompose(accAtBank2), "IBAN", accAtBank1.IBAN) + + getResponse(List(bank1Json, bank2SameIbanJson)).code should equal(FAILED) + + // And nothing partially imported. + Connector.connector.vend.getBankAccountLegacy(BankId(accAtBank1.bank), AccountId(accAtBank1.id), None).isDefined should equal(false) + Connector.connector.vend.getBankAccountLegacy(BankId(accAtBank2.bank), AccountId(accAtBank2.id), None).isDefined should equal(false) + + // The same two accounts import fine once their IBANs differ -- proving the rejection above + // was about the IBAN collision and not about anything else in the payload. + getResponse(List(bank1Json, Extraction.decompose(accAtBank2))).code should equal(SUCCESS) + Connector.connector.vend.getBankAccountLegacy(BankId(accAtBank1.bank), AccountId(accAtBank1.id), None).isDefined should equal(true) + Connector.connector.vend.getBankAccountLegacy(BankId(accAtBank2.bank), AccountId(accAtBank2.id), None).isDefined should equal(true) + } + it should "not allow an account to be created with an existing IBAN" in { val banks = standardBanks.map(Extraction.decompose) val users = standardUsers.map(Extraction.decompose) diff --git a/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala index 5e57ea2a7e..26d40aa469 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/ApiCollectionEndpointTest.scala @@ -27,6 +27,7 @@ package code.api.v4_0_0 import org.json4s._ import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.berlin.group.v1_3.Http4sBGv13Alias import code.api.util.APIUtil.OAuth._ import code.api.v4_0_0.APIMethods400.Implementations4_0_0 import com.github.dwickern.macros.NameOf.nameOf @@ -202,7 +203,79 @@ class ApiCollectionEndpointTest extends V400ServerSetup { val operationId= apiCollectionEndpoint.operation_id } - + + { + // Regression pin for the sandbox bug report: BGv2-getAccountDetails was served by the + // resource-docs dispatcher (/resource-docs/BGv2/obp) but missing from the global + // operation-id union getAllResourceDocs relies on, so this exact request used to fail + // with OBP-40048 Invalid operation_id. + Then(s"we test the $ApiEndpoint6- BGv2-getAccountDetails") + val requestApiCollectionEndpoint = (v4_0_0_Request / "my" / "api-collection-ids" / apiCollectionId / "api-collection-endpoints").POST <@ (user1) + + lazy val postApiCollectionEndpointJson = SwaggerDefinitionsJSON.postApiCollectionEndpointJson400.copy(operation_id="BGv2-getAccountDetails") + + val responseApiCollectionEndpointJson = makePostRequest(requestApiCollectionEndpoint, write(postApiCollectionEndpointJson)) + Then("We should get a 201") + responseApiCollectionEndpointJson.code should equal(201) + val apiCollectionEndpoint = responseApiCollectionEndpointJson.body.extract[ApiCollectionEndpointJson400] + + apiCollectionEndpoint.operation_id should be (postApiCollectionEndpointJson.operation_id) + apiCollectionEndpoint.api_collection_endpoint_id shouldNot be (null) + + val operationId= apiCollectionEndpoint.operation_id + } + + // Regression pin for the Berlin Group v1.3 alias gap: when berlin_group_v1_3_alias_path is + // set (0.6/v1 in test.default.props and in both CI workflows) Http4sBGv13Alias publishes + // re-stamped copies of the canonical BG v1.3 docs under their own operation ids -- served by + // the resource-docs dispatcher via ScannedApis discovery, but formerly missing from the + // global operation-id union, the same class of gap as BGv2 above. + // + // Guarded on the alias actually being configured, and the expected id is read back from its + // own docs rather than hard-coded: test.default.props is gitignored (.gitignore:21), so a + // fresh clone or an IDE runner may not carry that prop, and a deployment may configure a + // different path (which changes the id's prefix). + val aliasOperationId: Option[String] = Http4sBGv13Alias.resourceDocs + .find(_.partialFunctionName == "getPaymentInitiationStatus").map(_.operationId) + + aliasOperationId.foreach { opId => + Then(s"we test the $ApiEndpoint6- $opId (Berlin Group v1.3 alias)") + val requestApiCollectionEndpoint = (v4_0_0_Request / "my" / "api-collection-ids" / apiCollectionId / "api-collection-endpoints").POST <@ (user1) + + lazy val postApiCollectionEndpointJson = SwaggerDefinitionsJSON.postApiCollectionEndpointJson400.copy(operation_id = opId) + + val responseApiCollectionEndpointJson = makePostRequest(requestApiCollectionEndpoint, write(postApiCollectionEndpointJson)) + Then("We should get a 201") + responseApiCollectionEndpointJson.code should equal(201) + val apiCollectionEndpoint = responseApiCollectionEndpointJson.body.extract[ApiCollectionEndpointJson400] + + apiCollectionEndpoint.operation_id should be (postApiCollectionEndpointJson.operation_id) + apiCollectionEndpoint.api_collection_endpoint_id shouldNot be (null) + } + + { + // Regression pin for the third drift instance: the global operation-id union used to be + // built from the v6.0.0 aggregation, so operation ids belonging to endpoints that exist + // ONLY in v7.0.0 (getMyMetrics, getTopUsers, getTopConsumers) were absent from it and + // could not be added to an API collection either. getMyMetrics is v7-only -- it is not + // part of Http4sResourceDocAggregation.v600 -- so this pins the v7 base specifically, + // unlike the OBPv6.0.0-* cases above which passed even under the old v6-based union. + Then(s"we test the $ApiEndpoint6- OBPv7.0.0-getMyMetrics (v7-only endpoint)") + val requestApiCollectionEndpoint = (v4_0_0_Request / "my" / "api-collection-ids" / apiCollectionId / "api-collection-endpoints").POST <@ (user1) + + lazy val postApiCollectionEndpointJson = SwaggerDefinitionsJSON.postApiCollectionEndpointJson400.copy(operation_id="OBPv7.0.0-getMyMetrics") + + val responseApiCollectionEndpointJson = makePostRequest(requestApiCollectionEndpoint, write(postApiCollectionEndpointJson)) + Then("We should get a 201") + responseApiCollectionEndpointJson.code should equal(201) + val apiCollectionEndpoint = responseApiCollectionEndpointJson.body.extract[ApiCollectionEndpointJson400] + + apiCollectionEndpoint.operation_id should be (postApiCollectionEndpointJson.operation_id) + apiCollectionEndpoint.api_collection_endpoint_id shouldNot be (null) + + val operationId= apiCollectionEndpoint.operation_id + } + { Then(s"we test the $ApiEndpoint7") val requestGet = (v4_0_0_Request / "my" / "api-collection-ids" / apiCollectionId / "api-collection-endpoints").GET <@ (user1) @@ -213,7 +286,10 @@ class ApiCollectionEndpointTest extends V400ServerSetup { val apiCollectionsJsonGet400 = responseGet.body.extract[ApiCollectionEndpointsJson400] - apiCollectionsJsonGet400.api_collection_endpoints.length should be (4) + // Six unconditional cases above, plus the Berlin Group v1.3 alias one when that alias is + // configured for this run. + val expected = if (aliasOperationId.isDefined) 7 else 6 + apiCollectionsJsonGet400.api_collection_endpoints.length should be (expected) } } } diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala new file mode 100644 index 0000000000..72f6cc6934 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala @@ -0,0 +1,174 @@ +package code.api.v4_0_0 + +import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.util.ApiRole +import code.api.util.ErrorMessages.DynamicResourceDocMethodDependency +import code.entitlement.Entitlement +import com.openbankproject.commons.model.ErrorMessage +import org.json4s.native.Serialization.write + +/** + * With dynamic_code_compile_validate_enable=true, a Java method_body that calls an OBP method NOT + * on the dependency whitelist must be rejected -- proving createJavaHttp4sEndpoint validates the + * real compiled Java class (getCompiledInstance), not just its Scala wrapper. + */ +class DynamicResourceDocJavaSecurityValidationTest extends V400ServerSetup { + + private def maliciousMethodBody: String = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaSecurityProbe implements Supplier> { + | private Object apply(Object[] args) { + | // APIUtil.getPropsValue is NOT on dynamic_code_compile_validate_dependencies' + | // whitelist (only errorJsonResponse*/scalaFutureToLaFuture/futureToBoxedResponse are). + | String secret = code.api.util.APIUtil$.MODULE$.getPropsValue("hostname", "none"); + | Map response = new LinkedHashMap<>(); + | response.put("leaked", secret); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + // Mirrors sample.props.template's default dynamic_code_compile_validate_dependencies exactly, + // minus the trailing newlines/line-continuations -- deliberately does NOT list APIUtil.getPropsValue. + private def defaultDependenciesWhitelist: String = + """[NewStyle.function.getClass.getTypeName -> "*", CompiledObjects.getClass.getTypeName -> "sandbox", HttpCode.getClass.getTypeName -> "200", DynamicCompileEndpoint.getClass.getTypeName -> "getPathParams, scalaFutureToBoxedJsonResponse", APIUtil.getClass.getTypeName -> "errorJsonResponse, errorJsonResponse$default$1, errorJsonResponse$default$2, errorJsonResponse$default$3, errorJsonResponse$default$4, scalaFutureToLaFuture, futureToBoxedResponse", ErrorMessages.getClass.getTypeName -> "*", ExecutionContext.Implicits.getClass.getTypeName -> "global", JSONFactory400.getClass.getTypeName -> "createBanksJson", classOf[Sandbox].getTypeName -> "runInSandbox", classOf[CallContext].getTypeName -> "*", classOf[ResourceDoc].getTypeName -> "getPathParams", "scala.reflect.runtime.package$" -> "universe", PractiseEndpoint.getClass.getTypeName + "*" -> "*"]""" + + // Deliberately does NOT set show_used_connector_methods: that prop exists to opt in to an + // unrelated, expensive introspection/reporting feature and was never meant to gate security + // validation, which happens to reuse the same underlying bytecode scan. An operator who reads + // only dynamic_code_compile_validate_enable's own prop documentation and sets just these two + // props (as this method does) must still get real enforcement -- proving that is the point of + // every scenario below. + // + // Block body (not `= setPropsValues(...)`) so .github/scripts/check_test_isolation.py's brace + // scanner sees an opening `{` right after `def enableStrictValidation` and treats this as a + // safe "helper called from scenarios" scope rather than a class-body-level setPropsValues call. + private def enableStrictValidation(): Unit = { + setPropsValues( + "dynamic_code_compile_validate_enable" -> "true", + "dynamic_code_compile_validate_dependencies" -> defaultDependenciesWhitelist + ) + } + + // Every Java method_body implements Supplier> per convention (see + // DynamicUtil.createJavaHttp4sEndpoint's doc comment). javac always erases that generic + // Supplier.get() to a synthetic bridge method `Object get()` whose body just invokevirtual-calls + // the real, properly-typed get() -- an ordinary same-class call regardless of what the body does. + private def benignMethodBody: String = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaSecurityBenignProbe implements Supplier> { + | private Object apply(Object[] args) { + | Map response = new LinkedHashMap<>(); + | response.put("greeting", "hello"); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + private def createRequest = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + + feature("Security validation of Java method_body against dynamic_code_compile_validate_dependencies") { + + // Regression guard: the Supplier.get() generics-erasure bridge method's same-class call to the + // real get() must not itself be treated as a call to a forbidden method. Without this, every + // Java doc -- malicious or not -- was rejected under strict validation, because the compiled + // class lives under the OBP-owned code.* package but its randomly-generated name can never + // appear in a static whitelist. + scenario("Registering a benign Java doc succeeds even with strict validation enabled") { + enableStrictValidation() + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "benignProbeTest", + requestUrl = "/benign_probe_test/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(benignMethodBody, "UTF-8"), + programmingLang = "Java" + ) + val resp = makePostRequest(createRequest, write(doc)) + + Then("the compile succeeds -- the Supplier.get() bridge method's self-call is not a forbidden dependency") + resp.code should equal(201) + } + scenario("Registering a Java doc that calls a non-whitelisted OBP method is rejected with 400") { + enableStrictValidation() + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "securityProbeTest", + requestUrl = "/security_probe_test/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(maliciousMethodBody, "UTF-8"), + programmingLang = "Java" + ) + val resp = makePostRequest(createRequest, write(doc)) + + Then("the compile is rejected with 400 DynamicResourceDocMethodDependency, not accepted") + resp.code should equal(400) + resp.body.extract[ErrorMessage].message should include(DynamicResourceDocMethodDependency) + } + + // Regression guard for the bug createJavaHttp4sEndpoint had before it split compilation from + // validation: memoJavaCompiledScript (formerly memoJavaHttp4sEndpoint) memoized the WHOLE + // Box[Http4sEndpointIO], keyed only by the exact method_body string. A doc compiled once while + // validation was off got a cached Full(...) that a later, identical create call -- made AFTER + // validation was turned on and the whitelist tightened -- would silently reuse, never + // re-running Validation.validateDependency at all. This scenario reproduces exactly that + // sequence: compile the same malicious source once with validation off (succeeds, populates + // the compile cache), then enable strict validation and resubmit the identical source. + scenario("A Java source compiled once while validation was off is still validated on a later create call") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val firstDoc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "cacheBypassProbeTest1", + requestUrl = "/cache_bypass_probe_test_1/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(maliciousMethodBody, "UTF-8"), + programmingLang = "Java" + ) + When("validation is off (the suite default) and we compile the malicious source for the first time") + val firstResp = makePostRequest(createRequest, write(firstDoc)) + firstResp.code should equal(201) + + When("validation is then turned on with the same source resubmitted under a different doc") + enableStrictValidation() + val secondDoc = firstDoc.copy( + partialFunctionName = "cacheBypassProbeTest2", + requestUrl = "/cache_bypass_probe_test_2/MY_USER_ID" + ) + val secondResp = makePostRequest(createRequest, write(secondDoc)) + + Then("the second create is still rejected -- the compile-result cache must not bypass fresh validation") + secondResp.code should equal(400) + secondResp.body.extract[ErrorMessage].message should include(DynamicResourceDocMethodDependency) + } + } +} diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala new file mode 100644 index 0000000000..aa12c85bc8 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala @@ -0,0 +1,141 @@ +package code.api.v4_0_0 + +import org.json4s._ +import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.util.ApiRole +import code.api.util.ErrorMessages.DynamicCodeLangNotSupport +import code.dynamicResourceDoc.JsonDynamicResourceDoc +import code.entitlement.Entitlement +import com.openbankproject.commons.model.ErrorMessage +import com.openbankproject.commons.util.json +import org.json4s.native.JsonMethods.{compact, parse => parseJson, render} +import org.json4s.native.Serialization.write + +/** + * Java-language coverage for the DynamicResourceDoc runtime-compilation mechanism. + * DynamicResourceDocTest.scala covers the (unchanged) Scala-language path end-to-end; these + * scenarios exercise the new `programming_lang = "Java"` dispatch added to + * DynamicEndpoints.CompiledObjects / DynamicUtil.createJavaHttp4sEndpoint, plus the + * backward-compat and unsupported-language guards added alongside it. + */ +class DynamicResourceDocJavaTest extends V400ServerSetup { + + private def createDynamicResourceDocsRequest = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + + // Java-side convention: the pasted class implements Supplier>. + // args(0) = raw request body (String, or null), args(1) = path params (java.util.Map), + // args(2) = the CallContext. See DynamicUtil.createJavaHttp4sEndpoint's doc comment. + private def javaRoleTestMethodBody: String = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaResourceDocRoleTest implements Supplier> { + | private Object apply(Object[] args) { + | String rawBody = (String) args[0]; + | @SuppressWarnings("unchecked") + | Map pathParams = (Map) args[1]; + | String myUserId = pathParams.get("MY_USER_ID"); + | + | Map response = new LinkedHashMap<>(); + | response.put("user_id_from_path", myUserId + "_from_path"); + | response.put("received_body", rawBody); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + feature("Native execution of a runtime-compiled dynamic resource doc with a Java method_body") { + + scenario("Create a role-gated Java-language dynamic resource doc and verify 401 / 403 / 200") { + val dynamicRole = "CanCallJavaPieceCRoleTest" + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + When("We create a Java-language dynamic resource doc gated by that role") + val createReq = createDynamicResourceDocsRequest + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = dynamicRole, + partialFunctionName = "javaPieceCRoleTest", + requestUrl = "/my_java_role_user/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(javaRoleTestMethodBody, "UTF-8"), + programmingLang = "Java" + ) + val createResp = makePostRequest(createReq, write(doc)) + Then("We should get a 201") + createResp.code should equal(201) + createResp.body.extract[JsonDynamicResourceDoc].programmingLang should equal("Java") + + val callUrl = dynamicEndpoint_Request / "dynamic-resource-doc" / "my_java_role_user" / "user-1" + val body = """{"name":"Jhon","age":12,"hobby":["coding"]}""" + + assertRoleGated401Then403Then200(callUrl, body, dynamicRole) { resp200 => + val rendered = json.compactRender(resp200.body) + rendered should include("user-1_from_path") + rendered should include("Jhon") + } + } + + scenario("Reject an unsupported programming_lang before attempting compilation") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + When("We create a dynamic resource doc with an unsupported programming_lang") + val createReq = createDynamicResourceDocsRequest + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + partialFunctionName = "unsupportedLangTest", + requestUrl = "/unsupported_lang_test/MY_USER_ID", + programmingLang = "Python" + ) + val resp = makePostRequest(createReq, write(doc)) + + Then("We should get a 400 DynamicCodeLangNotSupport, not a compile-failure error") + resp.code should equal(400) + resp.body.extract[ErrorMessage].message should include(DynamicCodeLangNotSupport) + } + + scenario("Backward compatibility: a request body with programming_lang entirely omitted still creates a Scala-language doc") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + When("We create a dynamic resource doc from a JSON payload that predates the programming_lang field") + val posted = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "preExistingClientTest", + requestUrl = "/pre_existing_client_test/MY_USER_ID" + ) + // Simulate an old client payload: strip programming_lang out entirely rather than relying on + // Serialization.write, which always emits every case-class field (default or not). + val fullJson = parseJson(write(posted)) + val withoutLang = fullJson.removeField { case (name, _) => name == "programming_lang" } + val requestBodyStr = compact(render(withoutLang)) + requestBodyStr should not include "programming_lang" + + val createReq = createDynamicResourceDocsRequest + val createResp = makePostRequest(createReq, requestBodyStr) + + Then("We should get a 201 and the stored/served doc defaults to the Scala language") + createResp.code should equal(201) + createResp.body.extract[JsonDynamicResourceDoc].programmingLang should equal("Scala") + + Then("calling the endpoint still compiles and serves via the (unchanged) Scala template path") + val callReq = (dynamicEndpoint_Request / "dynamic-resource-doc" / "pre_existing_client_test" / "user-1").POST <@ (user1) + val callResp = makePostRequest(callReq, """{"name":"Jhon","age":12,"hobby":["coding"]}""") + callResp.code should equal(200) + val rendered = json.compactRender(callResp.body) + rendered should include("user-1_from_path") + rendered should include("Jhon") + } + } +} diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala index a482557a18..c7d21954af 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala @@ -29,7 +29,7 @@ import org.json4s._ import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON import code.api.util.APIUtil.OAuth._ import code.api.util.ApiRole._ -import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, DynamicResourceDocAlreadyExists, DynamicResourceDocNotFound, UserHasMissingRoles} +import code.api.util.ErrorMessages.{DynamicResourceDocAlreadyExists, DynamicResourceDocNotFound, UserHasMissingRoles} import code.api.util.ApiRole import code.api.v4_0_0.APIMethods400.Implementations4_0_0 import code.dynamicResourceDoc.JsonDynamicResourceDoc @@ -315,21 +315,9 @@ class DynamicResourceDocTest extends V400ServerSetup { val callUrl = dynamicEndpoint_Request / "dynamic-resource-doc" / "my_role_user" / "user-1" val body = """{"name":"Jhon","age":12,"hobby":["coding"]}""" - Then("calling without authentication returns 401") - val resp401 = makePostRequest(callUrl.POST, body) - resp401.code should equal(401) - resp401.body.extract[ErrorMessage].message should include(AuthenticatedUserIsRequired) - - Then("calling authenticated but without the role returns 403") - val resp403 = makePostRequest(callUrl.POST <@ (user1), body) - resp403.code should equal(403) - resp403.body.extract[ErrorMessage].message should include(UserHasMissingRoles) - - Then("granting the role makes the call succeed (200)") - Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, dynamicRole) - val resp200 = makePostRequest(callUrl.POST <@ (user1), body) - resp200.code should equal(200) - json.compactRender(resp200.body) should include("_from_path") + assertRoleGated401Then403Then200(callUrl, body, dynamicRole) { resp200 => + json.compactRender(resp200.body) should include("_from_path") + } } // Regression guard for DynamicEndpointCodeGenerator.buildTemplate: the template served by @@ -426,6 +414,42 @@ class DynamicResourceDocTest extends V400ServerSetup { storedRow.UpdatedByUserId.get should be(resourceUser1.userId) storedRow.MethodBodyHash.get should be(code.api.util.APIUtil.sha256Hex(URLDecoder.decode(changedMethodBody, "UTF-8"))) } + + // Regression guard: rows created before the Lang column existed have a genuine SQL NULL there + // (Schemifier's ALTER TABLE ADD COLUMN sets no default), not "Scala". An explicit null argument + // bypasses JsonDynamicResourceDoc's own programmingLang="Scala" default -- that default only + // applies when the argument is omitted entirely -- so a bare Lang.get would have surfaced as a + // null/empty programming_lang in the API response instead of falling back to "Scala". + scenario("A dynamic resource doc row predating the programming_lang column still reports \"Scala\"", ApiEndpoint1, ApiEndpoint3, VersionOfApi) { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canGetDynamicResourceDoc.toString) + + When("We create a dynamic resource doc") + val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val posted = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + partialFunctionName = "preDatesLangColumnTest", + requestUrl = "/pre_dates_lang_column_test/MY_USER_ID" + ) + val createResp = makePostRequest(createReq, write(posted)) + createResp.code should equal(201) + val docId = (createResp.body \ "dynamic_resource_doc_id").values.toString + + When("its lang column is forced to a genuine SQL NULL, bypassing the ORM (which always writes \"Scala\")") + import code.dynamicResourceDoc.DynamicResourceDoc + net.liftweb.mapper.DB.runUpdate( + s"UPDATE ${DynamicResourceDoc.dbTableName} SET ${DynamicResourceDoc.Lang.dbColumnName} = NULL " + + s"WHERE ${DynamicResourceDoc.DynamicResourceDocId.dbColumnName} = ?", + List(docId) + ) + + Then("GET still reports programming_lang as \"Scala\", not null or empty") + val getReq = (v4_0_0_Request / "management" / "dynamic-resource-docs" / docId).GET <@ (user1) + val getResp = makeGetRequest(getReq) + getResp.code should equal(200) + getResp.body.extract[JsonDynamicResourceDoc].programmingLang should equal("Scala") + } } } diff --git a/obp-api/src/test/scala/code/api/v4_0_0/Http4s400ViewResolutionTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/Http4s400ViewResolutionTest.scala new file mode 100644 index 0000000000..de94779a04 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v4_0_0/Http4s400ViewResolutionTest.scala @@ -0,0 +1,76 @@ +package code.api.v4_0_0 + +import code.api.util.CallContext +import com.openbankproject.commons.model.View +import net.liftweb.common.{Box, Empty} + +import scala.concurrent.Await +import scala.concurrent.duration._ + +/** + * `Http4s400.Implementations4_0_0.resolveCreateTransactionRequestView` decides between two + * failure shapes for `createTransactionRequest`'s view lookup: a genuinely missing view + * (client's fault, 404) and anything else the lookup throws (a connection-pool exhaustion, a + * transient SQL error, a Mapper bug -- none of them the client's fault, all of them a 500). + * + * `NewStyle.function.tryons` cannot tell these apart on its own: it catches any `Exception` the + * wrapped block raises and reports it via the caller-supplied failCode regardless of cause. If + * the whole DB lookup sits inside that block, an infra failure and a real not-found produce the + * identical JSON-encoded {"failCode":404,...} exception -- indistinguishable to + * ErrorResponseConverter, which resolves the 404 straight from that embedded field. A client + * whose retry logic reacts to 500 (retry) differently from 404 (stop) is told to stop when the + * backend is actually just broken. + * + * These tests call the production function directly with a stub `lookup`, so no live Mapper + * connection is needed to exercise the distinction -- but Implementations4_0_0's own static + * init registers the full v4.0.0 ResourceDoc set, which needs the app booted, hence + * V400ServerSetup rather than a bare unit-test base. + */ +class Http4s400ViewResolutionTest extends V400ServerSetup { + + private implicit val cc: CallContext = CallContext() + + private def resultOf[T](f: => T): Either[Throwable, T] = + try Right(f) catch { case t: Throwable => Left(t) } + + feature("createTransactionRequest's view lookup distinguishes not-found from infra failure") { + + scenario("a lookup that finds nothing fails with the JSON-encoded 404 envelope") { + val notFound: () => Box[View] = () => Empty + val outcome = resultOf(Await.result( + Http4s400.Implementations4_0_0.resolveCreateTransactionRequestView("nonexistent-view", notFound), + 5.seconds)) + + outcome match { + case Left(t) => + val msg = t.getMessage + withClue(s"expected a JSON envelope carrying failCode 404; got: $msg") { + msg should include("\"failCode\":404") + msg should include("View not found") + } + case Right(v) => + fail(s"expected the lookup to fail as not-found, but it returned $v") + } + } + + scenario("a lookup that throws for an infra reason propagates that exception, not a 404") { + val infraFailure = new RuntimeException("connection pool exhausted") + val broken: () => Box[View] = () => throw infraFailure + val outcome = resultOf(Await.result( + Http4s400.Implementations4_0_0.resolveCreateTransactionRequestView("some-view", broken), + 5.seconds)) + + outcome match { + case Left(t) => + val msg = t.getMessage + withClue(s"expected the original infra exception to propagate untouched so it resolves " + + s"to 500, not the 404 envelope reserved for a genuine not-found; got: $msg") { + msg should not include "\"failCode\":404" + msg should include("connection pool exhausted") + } + case Right(v) => + fail(s"expected the lookup's exception to propagate, but it returned $v") + } + } + } +} diff --git a/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala b/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala index ba69e99330..e33eee4eb5 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala @@ -21,9 +21,10 @@ import code.metadata.comments.MappedComment import code.metadata.narrative.MappedNarrative import code.metadata.transactionimages.MappedTransactionImage import code.metadata.wheretags.MappedWhereTag +import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, UserHasMissingRoles} import code.setup.{APIResponse, DefaultUsers, ServerSetupWithTestData} import code.transactionattribute.MappedTransactionAttribute -import com.openbankproject.commons.model.{AccountId, AccountRoutingJsonV121, AmountOfMoneyJsonV121, BankId, CreateViewJson, UpdateViewJSON} +import com.openbankproject.commons.model.{AccountId, AccountRoutingJsonV121, AmountOfMoneyJsonV121, BankId, CreateViewJson, ErrorMessage, UpdateViewJSON} import com.openbankproject.commons.util.ApiShortVersions import code.setup.OBPReq import org.json4s.native.Serialization.write @@ -42,6 +43,30 @@ trait V400ServerSetup extends ServerSetupWithTestData with DefaultUsers { def dynamicEndpoint_Request: OBPReq = baseRequest / "obp" / ApiShortVersions.`dynamic-endpoint`.toString def dynamicEntity_Request: OBPReq = baseRequest / "obp" / ApiShortVersions.`dynamic-entity`.toString + /** + * Shared by DynamicResourceDocTest and DynamicResourceDocJavaTest: exercises + * ResourceDoc.authCheckIO's role-gated path against a runtime-compiled dynamic-resource-doc -- + * calling without auth returns 401, authenticated without the role returns 403, and granting the + * role makes the call succeed (200), handed to `assertSuccess` for endpoint-specific checks. + */ + def assertRoleGated401Then403Then200(callUrl: OBPReq, body: String, dynamicRole: String)(assertSuccess: APIResponse => Unit): Unit = { + Then("calling without authentication returns 401") + val resp401 = makePostRequest(callUrl.POST, body) + resp401.code should equal(401) + resp401.body.extract[ErrorMessage].message should include(AuthenticatedUserIsRequired) + + Then("calling authenticated but without the role returns 403") + val resp403 = makePostRequest(callUrl.POST <@ (user1), body) + resp403.code should equal(403) + resp403.body.extract[ErrorMessage].message should include(UserHasMissingRoles) + + Then("granting the role makes the call succeed (200)") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, dynamicRole) + val resp200 = makePostRequest(callUrl.POST <@ (user1), body) + resp200.code should equal(200) + assertSuccess(resp200) + } + def randomBankId : String = { def getBanksInfo : APIResponse = { val request = v4_0_0_Request / "banks" diff --git a/obp-api/src/test/scala/code/api/v5_1_0/Http4s510JwtSignatureResolutionTest.scala b/obp-api/src/test/scala/code/api/v5_1_0/Http4s510JwtSignatureResolutionTest.scala new file mode 100644 index 0000000000..824353aac4 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v5_1_0/Http4s510JwtSignatureResolutionTest.scala @@ -0,0 +1,75 @@ +package code.api.v5_1_0 + +import code.api.util.CallContext + +import scala.concurrent.Await +import scala.concurrent.duration._ + +/** + * `Http4s510.Implementations5_1_0.resolveJwtSignatureValid` decides what a thrown exception from + * `JwtUtil.verifyJwt` means: a malformed client certificate or JWT (client's fault, 400) versus a + * JVM/security-provider configuration problem -- the requested signature algorithm is not + * registered (a hardened/FIPS JRE, a stripped provider list, a provider-registration bug). Both + * currently surface as the same `JOSEException`/generic `Exception`, and wrapping the whole call + * in `tryons(..., 400, ...)` cannot tell them apart: a security-provider fault would be reported + * to the caller as "your JSON is not signed" when nothing about their JWT is wrong -- the server + * cannot perform this verification for ANY caller until an operator fixes the JVM. + * + * These tests call the production function directly with a stub `verify`, so no real PEM/JWT + * material or JOSE library internals are needed to exercise the distinction. + */ +class Http4s510JwtSignatureResolutionTest extends V510ServerSetup { + + private implicit val cc: CallContext = CallContext() + + private def resultOf[T](f: => T): Either[Throwable, T] = + try Right(f) catch { case t: Throwable => Left(t) } + + feature("createConsumerDynamicRegistration's JWT verification distinguishes a bad client " + + "JWT from a broken security provider") { + + scenario("a verify() that returns false is a normal signature mismatch, not an error") { + val outcome = Await.result( + Http4s510.Implementations5_1_0.resolveJwtSignatureValid(() => false), 5.seconds) + outcome shouldBe false + } + + scenario("a verify() that throws for a malformed client JWT fails with the 400 envelope") { + val badJwt = new IllegalArgumentException("Invalid JWT serialization") + val outcome = resultOf(Await.result( + Http4s510.Implementations5_1_0.resolveJwtSignatureValid(() => throw badJwt), 5.seconds)) + + outcome match { + case Left(t) => + val msg = t.getMessage + withClue(s"expected a JSON envelope carrying failCode 400; got: $msg") { + msg should include("\"failCode\":400") + } + case Right(v) => + fail(s"expected verify() to fail as a bad JWT, but it returned $v") + } + } + + scenario("a verify() that throws because the JVM lacks the signature algorithm propagates " + + "that exception, not a 400") { + val providerFailure = + new com.nimbusds.jose.JOSEException("no such algorithm", + new java.security.NoSuchAlgorithmException("SHA256withRSA Signature not available")) + val outcome = resultOf(Await.result( + Http4s510.Implementations5_1_0.resolveJwtSignatureValid(() => throw providerFailure), 5.seconds)) + + outcome match { + case Left(t) => + val msg = t.getMessage + withClue(s"expected the original security-provider exception to propagate untouched " + + s"so it resolves to 500, not the 400 envelope reserved for a malformed " + + s"client JWT/certificate; got: $msg") { + msg should not include "\"failCode\":400" + msg should include("no such algorithm") + } + case Right(v) => + fail(s"expected verify()'s exception to propagate, but it returned $v") + } + } + } +} diff --git a/obp-api/src/test/scala/code/api/v6_0_0/Http4s600ResetPasswordPortalUrlTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/Http4s600ResetPasswordPortalUrlTest.scala new file mode 100644 index 0000000000..edd6e63973 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v6_0_0/Http4s600ResetPasswordPortalUrlTest.scala @@ -0,0 +1,52 @@ +package code.api.v6_0_0 + +import code.api.util.CallContext +import net.liftweb.common.{Empty, Full} + +import scala.concurrent.Await +import scala.concurrent.duration._ + +/** + * `Http4s600.Implementations6_0_0.resolveResetPasswordPortalUrl` reports what happens when + * `public_obp_portal_url` (or the legacy `portal_external_url`) isn't configured. That is an + * operator's configuration mistake, not a caller's -- the same condition this PR's own + * createTestEmail fix (Http4s700.scala) deliberately reports as 503, not 500, with the reasoning + * "the server is not broken -- it is not configured to do this, and [a wrong code] tells a + * caller with retry logic that the fault is transient." A missing portal URL should get the same + * treatment here: 503, not 400 -- an admin resetting a user's password should not be told their + * request was bad when the truth is nobody has configured the portal URL yet. + */ +class Http4s600ResetPasswordPortalUrlTest extends V600ServerSetup { + + private implicit val cc: CallContext = CallContext() + + private def resultOf[T](f: => T): Either[Throwable, T] = + try Right(f) catch { case t: Throwable => Left(t) } + + feature("resetPasswordUrl reports a missing portal URL as a server misconfiguration, not a client error") { + + scenario("a configured portal URL is used as-is") { + val url = Await.result( + Http4s600.Implementations6_0_0.resolveResetPasswordPortalUrl(Full("https://portal.example.com")), + 5.seconds) + url shouldBe "https://portal.example.com" + } + + scenario("an unconfigured portal URL fails with 503, not 400") { + val outcome = resultOf(Await.result( + Http4s600.Implementations6_0_0.resolveResetPasswordPortalUrl(Empty), 5.seconds)) + + outcome match { + case Left(t) => + val msg = t.getMessage + withClue(s"expected a JSON envelope carrying failCode 503 (an operator configuration " + + s"problem, not the caller's fault); got: $msg") { + msg should include("\"failCode\":503") + msg should not include "\"failCode\":400" + } + case Right(v) => + fail(s"expected the missing portal URL to fail, but it returned $v") + } + } + } +} diff --git a/obp-api/src/test/scala/code/api/v6_0_0/MetricAuthTypeTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/MetricAuthTypeTest.scala new file mode 100644 index 0000000000..1a79eb22f6 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v6_0_0/MetricAuthTypeTest.scala @@ -0,0 +1,60 @@ +package code.api.v6_0_0 + +import code.api.util.APIUtil.OAuth._ +import code.api.util.ApiRole.CanReadMetrics +import code.entitlement.Entitlement +import code.metrics.MetricBatchWriter +import com.openbankproject.commons.util.ApiVersion +import org.scalatest.Tag + +/** + * Tests the auth_type field on metric rows (v6.0.0): the authentication SCHEME of each + * call — never the credential. Uses the url-filter isolation pattern (write_metrics is + * JVM-wide; other suites also land rows). + * + * The harness's <@ (user1) authenticates via DirectLogin, so this suite's rows must + * carry auth_type "DirectLogin"; the invariant "Consent only with a consent reference" + * is asserted over whatever rows the filter returns. + */ +class MetricAuthTypeTest extends V600ServerSetup { + + object VersionOfApi extends Tag(ApiVersion.v6_0_0.toString) + object ApiEndpoint1 extends Tag("getMetrics") + + private val KnownAuthTypes = + Set("Consent", "OAuth2", "OAuth1", "DirectLogin", "GatewayLogin", "DAuth", "Anonymous", "Other") + + feature(s"test auth_type on metric rows version $VersionOfApi") { + scenario("Rows carry the scheme that authenticated them", ApiEndpoint1, VersionOfApi) { + setPropsValues("write_metrics" -> "true") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanReadMetrics.toString) + + val trafficUrl = "/obp/v5.1.0/banks" + val requestBanks = (v5_1_0_Request / "banks").GET <@ (user1) + (1 to 3).foreach(_ => makeGetRequest(requestBanks)) + + MetricBatchWriter.flush() + + When("We fetch the metric rows for the traffic url") + val request = (v6_0_0_Request / "management" / "metrics").GET <@ (user1) < + val authType = (m \ "auth_type").extractOpt[String] + withClue(s"row: $m ") { + authType.isDefined shouldBe true + KnownAuthTypes should contain(authType.get) + // Scheme/reference consistency: "Consent" implies a consent reference and vice versa. + val consentRef = (m \ "consent_reference_id").extractOpt[String] + (authType.get == "Consent") shouldBe consentRef.isDefined + } + } + metrics.map(m => (m \ "auth_type").extractOpt[String]).flatten.toSet shouldBe Set("DirectLogin") + } + } +} diff --git a/obp-api/src/test/scala/code/api/v6_0_0/ValidateDynamicResourceDocTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/ValidateDynamicResourceDocTest.scala new file mode 100644 index 0000000000..7637619c6b --- /dev/null +++ b/obp-api/src/test/scala/code/api/v6_0_0/ValidateDynamicResourceDocTest.scala @@ -0,0 +1,81 @@ +package code.api.v6_0_0 + +import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.util.ApiRole +import code.entitlement.Entitlement +import org.json4s.native.Serialization.write + +/** + * `POST /obp/v6.0.0/management/dynamic-resource-docs/validate` must reject an unsupported + * `programming_lang` the same way `POST .../dynamic-resource-docs` (create) does, rather than + * reporting `valid = true` for a language create would actually 400 on -- see + * Http4s600.validateDynamicResourceDoc's own doc comment: CompiledObjects falls through to the + * Scala compile path for any programming_lang value it doesn't recognise as Java, so a body that + * happens to be valid Scala would otherwise "validate" successfully under a bogus/misspelled + * language. + */ +class ValidateDynamicResourceDocTest extends V600ServerSetup { + + private def validateRequest = (v6_0_0_Request / "management" / "dynamic-resource-docs" / "validate").POST <@ (user1) + + private def docWith(programmingLang: String) = + SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + programmingLang = programmingLang + ) + + feature("Validate Dynamic Resource Doc rejects an unsupported programming_lang") { + scenario("An unsupported programming_lang is rejected, not silently validated as Scala") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val resp = makePostRequest(validateRequest, write(docWith("Python"))) + + Then("the request is rejected with 400, not a 200 valid=true/false body") + resp.code should equal(400) + resp.body.toString should include("OBP-40049") + } + + scenario("programming_lang \"Scala\" is accepted (baseline, unaffected by the language check)") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val resp = makePostRequest(validateRequest, write(docWith("Scala"))) + + Then("the request reaches the compile step and responds 200") + resp.code should equal(200) + (resp.body \ "valid").values should equal(true) + } + + scenario("programming_lang \"Java\" is accepted (baseline, unaffected by the language check)") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val javaBody = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class ValidateEndpointJavaProbe implements Supplier> { + | private Object apply(Object[] args) { + | Map response = new LinkedHashMap<>(); + | response.put("greeting", "hello"); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + val doc = docWith("Java").copy(methodBody = java.net.URLEncoder.encode(javaBody, "UTF-8")) + val resp = makePostRequest(validateRequest, write(doc)) + + Then("the request reaches the compile step and responds 200") + resp.code should equal(200) + (resp.body \ "valid").values should equal(true) + } + } +} diff --git a/obp-api/src/test/scala/code/concurrency/ConcurrentRateLimiterRaceTest.scala b/obp-api/src/test/scala/code/concurrency/ConcurrentRateLimiterRaceTest.scala index f3a3d1c205..d84a2ea045 100644 --- a/obp-api/src/test/scala/code/concurrency/ConcurrentRateLimiterRaceTest.scala +++ b/obp-api/src/test/scala/code/concurrency/ConcurrentRateLimiterRaceTest.scala @@ -2,6 +2,7 @@ package code.concurrency import code.api.JedisMethod import code.api.cache.Redis +import code.setup.RedisTestTarget import java.util.UUID import java.util.concurrent.atomic.AtomicInteger @@ -31,7 +32,7 @@ class ConcurrentRateLimiterRaceTest extends ConcurrentRaceSetup { feature("Redis-backed rate-limit and idempotency operations must be atomic") { scenario("H4: concurrent check-then-increment must not let more than `limit` callers pass the gate", ConcurrencyRace) { - assume(redisUp, "Redis not reachable — skipping H4") + RedisTestTarget.requireReachable(redisUp, "H4") Given("a rate-limit counter key with limit=5 and 20 concurrent callers") val key = "__conc_h4_rl_" + UUID.randomUUID.toString.take(8) val limit = 5L @@ -67,7 +68,7 @@ class ConcurrentRateLimiterRaceTest extends ConcurrentRaceSetup { } scenario("M6: idempotency response cache must be first-write-wins, not last-writer-wins (SET NX EX, not setex)", ConcurrencyRace) { - assume(redisUp, "Redis not reachable — skipping M6") + RedisTestTarget.requireReachable(redisUp, "M6") Given("an idempotency response key that receives two writes with different bodies") val key = "__conc_m6_rd_" + UUID.randomUUID.toString.take(8) val ttl = 60 @@ -90,7 +91,7 @@ class ConcurrentRateLimiterRaceTest extends ConcurrentRaceSetup { } scenario("M7: idempotency lock must be acquired atomically with its TTL (SET NX EX, not setnx+expire)", ConcurrencyRace) { - assume(redisUp, "Redis not reachable — skipping M7") + RedisTestTarget.requireReachable(redisUp, "M7") Given("a lock key acquired the way IdempotencyMiddleware.tryAcquireLock now does it") val key = "__conc_m7_lock_" + UUID.randomUUID.toString.take(8) val lockTtl = 60 diff --git a/obp-api/src/test/scala/code/metrics/MetricsTest.scala b/obp-api/src/test/scala/code/metrics/MetricsTest.scala index 729abd8cdd..28f468ed59 100644 --- a/obp-api/src/test/scala/code/metrics/MetricsTest.scala +++ b/obp-api/src/test/scala/code/metrics/MetricsTest.scala @@ -65,7 +65,7 @@ class MetricsTest extends ServerSetup with WipeMetrics { scenario("We save a new API metric") { metrics.saveMetric(testUserId,testUrl1, day1, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) MetricBatchWriter.flush() val byUrl = metrics.getAllMetrics(List(OBPLimit(limit))).groupBy(_.getUrl()) @@ -83,16 +83,16 @@ class MetricsTest extends ServerSetup with WipeMetrics { scenario("Group all metrics by url") { metrics.saveMetric(testUserId, testUrl1, day1, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) metrics.saveMetric(testUserId, testUrl1, day1, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) metrics.saveMetric(testUserId, testUrl1, day2, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) metrics.saveMetric(testUserId, testUrl2, day2, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) MetricBatchWriter.flush() val byUrl = metrics.getAllMetrics(List(OBPLimit(limit1))).groupBy(_.getUrl()) @@ -113,16 +113,16 @@ class MetricsTest extends ServerSetup with WipeMetrics { scenario("Group all metrics by day") { metrics.saveMetric(testUserId, testUrl1, day1, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) metrics.saveMetric(testUserId, testUrl1, day1, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) metrics.saveMetric(testUserId, testUrl1, day2, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) metrics.saveMetric(testUserId, testUrl2, day2, -1L, testUserName, testAppName, testDeveloperEmail, testConsumerId, testImplementedByPartialFunction, - testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null) + testVersion, testVerb, None, getCorrelationId(), testResponseBody, testSourceIp , testTargetIp, testApiInstanceId, null, null, null, null) MetricBatchWriter.flush() val byDay = metrics.getAllMetrics(List(OBPLimit(limit2))).groupBy(APIMetrics.getMetricDay) diff --git a/obp-api/src/test/scala/code/setup/RedisTestTarget.scala b/obp-api/src/test/scala/code/setup/RedisTestTarget.scala new file mode 100644 index 0000000000..9d7ba236b1 --- /dev/null +++ b/obp-api/src/test/scala/code/setup/RedisTestTarget.scala @@ -0,0 +1,49 @@ +package code.setup + +import org.scalatest.Assertions + +/** + * Whether a Redis-dependent check may cancel itself, or has to run. + * + * `assume(Redis.isRedisReady)` cancels wherever no Redis is listening, and a cancelled check is + * indistinguishable from a passing one in every report anybody reads. and `run_tests_parallel.sh` starts none and probes for none, so the rate-limiter races + * and the cache-invalidation checks skip on every local run while reporting green. CI does declare + * a redis service, but nothing fails if that block is dropped or the container never becomes + * healthy - the suite would go back to skipping, silently, and the log line saying so is one nobody + * reads. + * + * What is lost when they skip is not incidental: these are the only checks that exercise the + * rate-limiter's Redis fast path and MethodRouting's cache invalidation under concurrency. Both are + * shared-state races, which is precisely the class of defect a green unit suite cannot rule out. + * + * `OBP_TEST_REDIS_REQUIRED=true` turns the cancellation into a failure. CI sets it alongside the + * service container; developers leave it unset and keep the skip. + */ +object RedisTestTarget { + + /** True when a missing Redis must fail rather than cancel. */ + def required: Boolean = + sys.env.get("OBP_TEST_REDIS_REQUIRED").exists(_.trim.equalsIgnoreCase("true")) + + /** + * Cancel the test when Redis is absent and optional; fail it when absent and required; return + * normally when it is there. + * + * `required` is a parameter rather than a direct read of the environment for the same reason it + * is one on PostgresTestTarget: the environment cannot be changed from inside a running JVM, so a + * branch that only an environment variable can reach is a branch no test can enter - and an + * unreachable branch in a guard is the very thing the guard exists to stop. + */ + def requireReachable(reachable: Boolean, what: String, required: Boolean = required): Unit = + if (!reachable) { + if (required) { + Assertions.fail( + s"OBP_TEST_REDIS_REQUIRED=true but Redis is not reachable, so $what cannot run. These " + + "checks are the only cover for the rate limiter's Redis path and MethodRouting cache " + + "invalidation under concurrency, so they must not be skipped where they are required - " + + "start Redis, or unset OBP_TEST_REDIS_REQUIRED to go back to skipping.") + } else { + Assertions.cancel(s"Redis not reachable - skipping $what") + } + } +} diff --git a/obp-api/src/test/scala/code/setup/RedisTestTargetTest.scala b/obp-api/src/test/scala/code/setup/RedisTestTargetTest.scala new file mode 100644 index 0000000000..3d7d374c8c --- /dev/null +++ b/obp-api/src/test/scala/code/setup/RedisTestTargetTest.scala @@ -0,0 +1,53 @@ +package code.setup + +import org.scalatest.{FlatSpec, Matchers} +import org.scalatest.exceptions.{TestCanceledException, TestFailedException} + +/** + * The gate itself, both ways. + * + * PostgresTestTargetTest exists for the same reason and states it plainly: the skip has to be + * opt-in, because a skip reports as a pass. A guard whose strict branch is never entered by any + * test is a guard nobody has checked, and this one's whole purpose is to stop exactly that shape + * of untested branch. + * + * `required` is passed explicitly rather than read from the environment, because the environment + * cannot be changed from inside a running JVM -- which is why RedisTestTarget takes it as a + * parameter in the first place. + */ +class RedisTestTargetTest extends FlatSpec with Matchers { + + /** Arbitrary label passed as `what`; only its identity across calls matters, not its text. */ + private val CheckLabel = "a check" + + "requireReachable" should "return normally when Redis is reachable, whether or not it is required" in { + noException should be thrownBy RedisTestTarget.requireReachable( + reachable = true, what = CheckLabel, required = false) + noException should be thrownBy RedisTestTarget.requireReachable( + reachable = true, what = CheckLabel, required = true) + } + + it should "cancel when Redis is absent and optional" in { + val e = intercept[TestCanceledException] { + RedisTestTarget.requireReachable(reachable = false, what = CheckLabel, required = false) + } + e.getMessage should include("Redis not reachable") + e.getMessage should include(CheckLabel) + } + + it should "FAIL, not cancel, when Redis is absent and required" in { + val e = intercept[TestFailedException] { + RedisTestTarget.requireReachable(reachable = false, what = CheckLabel, required = true) + } + e.getMessage should include("OBP_TEST_REDIS_REQUIRED=true") + e.getMessage should include(CheckLabel) + } + + "required" should "read OBP_TEST_REDIS_REQUIRED and default to false" in { + // Whatever this environment says, the value has to be a Boolean derived from that one + // variable -- and unset must mean false, so a developer machine keeps the skip. + val fromEnv = sys.env.get("OBP_TEST_REDIS_REQUIRED").exists(_.trim.equalsIgnoreCase("true")) + RedisTestTarget.required shouldBe fromEnv + if (sys.env.get("OBP_TEST_REDIS_REQUIRED").isEmpty) RedisTestTarget.required shouldBe false + } +} diff --git a/obp-api/src/test/scala/code/util/DynamicUtilTest.scala b/obp-api/src/test/scala/code/util/DynamicUtilTest.scala index c1bfd22e2b..c9be8d2452 100644 --- a/obp-api/src/test/scala/code/util/DynamicUtilTest.scala +++ b/obp-api/src/test/scala/code/util/DynamicUtilTest.scala @@ -48,6 +48,35 @@ class DynamicUtilTest extends FlatSpec with Matchers { private val securityManagerUnavailable = "SecurityManager enforcement is not available on JDK 17+ (JEP 411); skip on JDK 21" + /** + * Skip the sandbox checks when no SecurityManager can enforce them -- but let CI refuse the skip. + * + * `assume` cancels, and a cancelled check is indistinguishable from a passing one in every + * report anybody reads: this suite has shown "canceled 0" while three of its scenarios never + * ran, because ScalaTest counts a cancellation separately from a failure and the summary line + * people look at is the failure count. Since JDK 17 removed SecurityManager enforcement + * (JEP 411, completed by JEP 486), DynamicUtil.Sandbox is a no-op on any modern JDK and these + * three have been skipping on every run, everywhere, for as long as the build has been on 21+. + * + * That is a real gap, not a formality: the sandbox is what stops runtime-compiled endpoint code + * from reading the filesystem or opening sockets, and nothing else covers it. + * + * OBP_TEST_SANDBOX_REQUIRED=true turns the cancellation into a failure, the same lever + * RedisTestTarget gives the Redis-dependent checks. Set it wherever a JDK that can still + * enforce is available; leave it unset and the skip stands, but now it is a decision somebody + * made rather than a silence. + */ + private def requireSecurityManager(): Unit = + if (System.getSecurityManager == null) { + val required = sys.env.get("OBP_TEST_SANDBOX_REQUIRED").exists(_.trim.equalsIgnoreCase("true")) + if (required) + fail("OBP_TEST_SANDBOX_REQUIRED=true but no SecurityManager is installed, so the sandbox " + + "checks cannot run. They are the only cover for what runtime-compiled endpoint code " + + "is allowed to touch -- run them on a JDK that still enforces, or unset the variable " + + "to go back to skipping.") + else cancel(securityManagerUnavailable) + } + implicit val formats = code.api.util.CustomJsonFormats.formats @@ -115,14 +144,18 @@ class DynamicUtilTest extends FlatSpec with Matchers { val dependenciesString = """[NewStyle.function.getClass.getTypeName -> "*",CompiledObjects.getClass.getTypeName -> "sandbox",HttpCode.getClass.getTypeName -> "200",DynamicCompileEndpoint.getClass.getTypeName -> "getPathParams, scalaFutureToBoxedJsonResponse",APIUtil.getClass.getTypeName -> "errorJsonResponse, errorJsonResponse$default$1, errorJsonResponse$default$2, errorJsonResponse$default$3, errorJsonResponse$default$4, scalaFutureToLaFuture, futureToBoxedResponse",ErrorMessages.getClass.getTypeName -> "*",ExecutionContext.Implicits.getClass.getTypeName -> "global",JSONFactory400.getClass.getTypeName -> "createBanksJson",classOf[Sandbox].getTypeName -> "runInSandbox",classOf[CallContext].getTypeName -> "*",classOf[ResourceDoc].getTypeName -> "getPathParams","scala.reflect.runtime.package$" -> "universe",PractiseEndpoint.getClass.getTypeName + "*" -> "*"]""".stripMargin - val scalaCode2 = s"${DynamicUtil.importStatements}"+dependenciesString.replaceFirst("\\[","Map[String, String](").dropRight(1) +").mapValues(v => StringUtils.split(v, ',').map(_.trim).toSet).toMap" + // DynamicUtil.Validation.dependenciesScalaCode, not a copy of it. This line used to be a + // character-for-character duplicate of the production expression, which meant an edit to + // either one left the test green while the two disagreed -- and this is the only compile that + // happens reflectively at boot, so nothing at compile time would have noticed either. + val scalaCode2 = DynamicUtil.Validation.dependenciesScalaCode(dependenciesString) val dependenciesBox2: Box[Map[String, Set[String]]] = DynamicUtil.compileScalaCode(scalaCode2) val dependencies2 = dependenciesBox2.openOrThrowException("Can not compile the string to Map") dependencies2.toString contains ("code.api.util.NewStyle") shouldBe (true) } "Sandbox.createSandbox method" should "should throw exception" taggedAs DynamicUtilsTag in { - assume(System.getSecurityManager != null, securityManagerUnavailable) + requireSecurityManager() val permissionList = List( // new java.net.SocketPermission("ir.dcs.gla.ac.uk:80","connect,resolve"), ) @@ -147,7 +180,7 @@ class DynamicUtilTest extends FlatSpec with Matchers { } "Sandbox.sandbox method test bankId" should "should throw exception" taggedAs DynamicUtilsTag in { - assume(System.getSecurityManager != null, securityManagerUnavailable) + requireSecurityManager() intercept[AccessControlException] { Sandbox.sandbox(bankId= "abc").runInSandbox { BankId("123" ) @@ -162,7 +195,7 @@ class DynamicUtilTest extends FlatSpec with Matchers { } "Sandbox.sandbox method test default permission" should "should throw exception" taggedAs DynamicUtilsTag in { - assume(System.getSecurityManager != null, securityManagerUnavailable) + requireSecurityManager() intercept[AccessControlException] { Sandbox.sandbox(bankId= "abc").runInSandbox { scala.io.Source.fromURL("https://apisandbox.openbankproject.com/") diff --git a/obp-api/src/test/scala/code/util/RunTestsParallelScriptTest.scala b/obp-api/src/test/scala/code/util/RunTestsParallelScriptTest.scala new file mode 100644 index 0000000000..cbfb155bba --- /dev/null +++ b/obp-api/src/test/scala/code/util/RunTestsParallelScriptTest.scala @@ -0,0 +1,61 @@ +package code.util + +import org.scalatest.{FlatSpec, Matchers} + +import java.io.File +import scala.io.Source + +/** + * Pins that run_tests_parallel.sh's zero-test-floor diagnostic message quotes the same number + * it actually compares against. + * + * The floor check reads: + * + * if [[ "${SF_TOTAL:-0}" -lt 3200 ]]; then + * echo " ✗ suspicious total: only ${SF_TOTAL:-0} tests ran (< 2000 floor) ..." + * + * The threshold was raised from 2000 to 3200 (see the script's own comment: "3200 is 90% of the + * 3571 measured on develop-obp") but the message text was not updated alongside it. A run that + * produces, say, 2500 tests is correctly failed by the `-lt 3200` check, but the printed + * diagnostic reads "only 2500 tests ran (< 2000 floor)" -- which is arithmetically + * self-contradictory (2500 is not less than 2000) to whoever is reading the CI log to work out + * why the build failed. + * + * A runtime test cannot exercise a bash script's own comparison, so this reads the script's + * source and asserts the number in the `-lt` comparison matches the number quoted in the message + * -- the same drift-guard shape SweepCoverageDriftCheckTest uses for a Scala file. + */ +class RunTestsParallelScriptTest extends FlatSpec with Matchers { + + private def scriptSource: String = { + val candidates = List( + new File("run_tests_parallel.sh"), + new File("../run_tests_parallel.sh") + ) + val file = candidates.find(_.isFile).getOrElse( + fail(s"Cannot locate run_tests_parallel.sh under either candidate path - this guard must " + + s"not pass by failing to look. Tried: ${candidates.mkString(", ")}")) + val source = Source.fromFile(file, "UTF-8") + try source.mkString finally source.close() + } + + "the zero-test floor diagnostic" should "quote the same threshold it actually compares against" in { + val src = scriptSource + + val comparisonThreshold = """\$\{SF_TOTAL:-0\}"\s*-lt\s*(\d+)""".r + .findFirstMatchIn(src).map(_.group(1)).getOrElse( + fail("could not find the zero-test floor comparison (\"${SF_TOTAL:-0}\" -lt N) in the " + + "script - this guard must not pass by failing to look")) + + val messageThreshold = """\(<\s*(\d+)\s+floor\)""".r + .findFirstMatchIn(src).map(_.group(1)).getOrElse( + fail("could not find the \"(< N floor)\" diagnostic text in the script - this guard " + + "must not pass by failing to look")) + + withClue(s"the script compares against $comparisonThreshold but tells the reader the floor " + + s"is $messageThreshold -- whichever one is stale, a CI failure reads as " + + s"self-contradictory until they match: ") { + messageThreshold shouldBe comparisonThreshold + } + } +} diff --git a/run_tests_parallel.sh b/run_tests_parallel.sh index f9d94aa7c8..b37ff9e29a 100755 --- a/run_tests_parallel.sh +++ b/run_tests_parallel.sh @@ -516,7 +516,17 @@ while IFS= read -r _f; do fi _fa=$(_sf_attr "$_head" failures); _fa=${_fa:-0} _e=$(_sf_attr "$_head" errors); _e=${_e:-0} - _sk=$(_sf_attr "$_head" skipped); _sk=${_sk:-0} + # ScalaTest's JUnit XML reporter does NOT put a skipped="N" attribute on ; + # it emits a child inside each cancelled . Reading the attribute + # therefore always yielded 0, so this line reported "0 skipped/canceled" for a run in + # which DynamicUtilTest cancelled three of its nine -- and would have reported the same + # for a suite that cancelled every one of its tests. That is the number somebody checks + # precisely when they suspect tests are not running, so it has to be counted from what + # is actually in the file. Attribute first for reporters that do emit it, child elements + # otherwise. + _sk=$(_sf_attr "$_head" skipped) + if [[ -z "$_sk" ]]; then _sk=$(grep -c "/dev/null); fi + _sk=${_sk:-0} SF_TOTAL=$((SF_TOTAL+_t)); SF_FAIL=$((SF_FAIL+_fa)); SF_ERR=$((SF_ERR+_e)); SF_SKIP=$((SF_SKIP+_sk)) if [[ $_fa -ne 0 || $_e -ne 0 ]]; then SF_BAD+=("$(basename "$_f" | sed 's/^TEST-//; s/\.xml$//'): $_fa failed, $_e errors") @@ -529,10 +539,16 @@ if [[ "$SF_FAIL" != "0" ]] || [[ "$SF_ERR" != "0" ]] || [[ "$SF_BROKEN" != "0" ] OVERALL_RC=1 fi # Zero-test floor: -DfailIfNoTests=false means a broken wildcardSuites filter runs nothing -# and "passes". The suite has ~2900 tests; a total far below that means shards ran -# near-empty — fail instead of reporting a hollow green. -if [[ "${SF_TOTAL:-0}" -lt 2000 ]]; then - echo " ✗ suspicious total: only ${SF_TOTAL:-0} tests ran (< 2000 floor) — filter/discovery regression?" +# and "passes". A total far below the real one means shards ran near-empty — fail instead of +# reporting a hollow green. +# +# 3200 is 90% of the 3571 measured on develop-obp (2026-08-25, --shards=4). The previous +# figure, 2000, was set against a suite the header called "~2900" and had drifted far enough +# that a run losing a fifth of its tests would still have passed it. Re-measure and re-set +# both numbers when the suite grows: a floor that is only half the real count is barely a +# floor at all. +if [[ "${SF_TOTAL:-0}" -lt 3200 ]]; then + echo " ✗ suspicious total: only ${SF_TOTAL:-0} tests ran (< 3200 floor) — filter/discovery regression?" OVERALL_RC=1 fi