Merge pull request #2736 from simonredfern/develop

Account Access performance Improvements and tests. Dynamic Consumer creation from Consent tweaks and logging.

Author: simonredfern

obp-api/src/main/scala/code/api/OAuth2.scala

@@ -501,15 +501,10 @@ object OAuth2Login extends RestHelper with MdcLoggable {
       val sub = getClaim(name = "sub", jwtToken = jwtToken)
       val email = getClaim(name = "email", jwtToken = jwtToken)
       val name = getClaim(name = "name", jwtToken = jwtToken).orElse(description)
-      val consumerId = azp match {
-        case Some(value) if APIUtil.checkIfStringIsUUID(value) => azp
-        case Some(value) => Some(s"${value}_${APIUtil.generateUUID()}")
-        case None => Some(APIUtil.generateUUID())
-      }
       Consumers.consumers.vend.getOrCreateConsumer(
-        consumerId = consumerId, // Use azp as consumer id if it is uuid value
-        key = Some(Helpers.randomString(40).toLowerCase),
-        secret = Some(Helpers.randomString(40).toLowerCase),
+        consumerId = None,
+        key = None,
+        secret = None,
         aud = aud,
         azp = azp,
         iss = iss,

obp-api/src/main/scala/code/api/openidconnect.scala

@@ -306,9 +306,9 @@ object OpenIdConnect extends OBPRestHelper with MdcLoggable {
 
   private def getOrCreateConsumer(idToken: String, userId: String): Box[Consumer] = {
     Consumers.consumers.vend.getOrCreateConsumer(
-      consumerId=Some(APIUtil.generateUUID()),
-      Some(Helpers.randomString(40).toLowerCase),
-      Some(Helpers.randomString(40).toLowerCase),
+      consumerId=None,
+      None,
+      None,
       Some(JwtUtil.getAudience(idToken).mkString(",")),
       getClaim(name = "azp", idToken = idToken),
       JwtUtil.getIssuer(idToken),

obp-api/src/main/scala/code/api/util/APIUtil.scala

@@ -3170,7 +3170,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
         // This is normal for certificate-based validation or anonymous requests
         Empty
     }
-    logger.debug(s"getUserAndSessionContextFuture says consumerByConsumerKey is: $consumerByConsumerKey")
+    //logger.debug(s"getUserAndSessionContextFuture says consumerByConsumerKey is: $consumerByConsumerKey")
 
     val res =
       if (authHeadersWithEmptyValues.nonEmpty) { // Check Authorization Headers Empty Values

obp-api/src/main/scala/code/model/OAuth.scala

@@ -394,31 +394,83 @@ object MappedConsumersProvider extends ConsumersProvider with MdcLoggable {
                                    logoUrl: Option[String],
                                   ): Box[Consumer] = {
 
-    val consumer: Box[Consumer] =
-      // 1st try to find via UUID issued by OBP-API back end
-      Consumer.find(By(Consumer.consumerId, consumerId.getOrElse("None"))) or
-        // 2nd try to find via the pair (azp, iss) issued by External Identity Provider
-        {
-          // The azp field in the payload of a JWT (JSON Web Token) represents the Authorized Party.
-          // It is typically used in the context of OAuth 2.0 and OpenID Connect to identify the client application that the token is issued for.
-          // The pair (azp, iss) is a unique key in case of Client of an Identity Provider
-          Consumer.find(By(Consumer.azp, azp.getOrElse("None")), By(Consumer.iss, iss.getOrElse("None")))
+    logger.info(s"getOrCreateConsumer says: BEGIN lookup. Input: consumerId=${consumerId.getOrElse("None")}, azp=${azp.getOrElse("None")}, iss=${iss.getOrElse("None")}, sub=${sub.getOrElse("None")}")
+
+    // 1st try: find by consumerId (UUID issued by OBP-API back end)
+    val byConsumerId = Consumer.find(By(Consumer.consumerId, consumerId.getOrElse("None")))
+    val consumer: Box[Consumer] = if (byConsumerId.isDefined) {
+      val c = byConsumerId.openOrThrowException("checked isDefined")
+      logger.info(s"getOrCreateConsumer says: MATCH on lookup 1 (by consumerId). Found consumer: consumerId=${c.consumerId.get}, key=${c.key.get}, azp=${c.azp.get}, iss=${c.iss.get}")
+      byConsumerId
+    } else {
+      logger.info(s"getOrCreateConsumer says: MISS on lookup 1 (by consumerId=${consumerId.getOrElse("None")}). Trying lookup 2 (by Consumer.key matching azp)...")
+
+      // 2nd try: find by consumer key matching azp (pre-registered consumer whose key is the OAuth2 client_id)
+      // This is checked before (azp, iss) so that a pre-registered consumer takes priority over an auto-created one
+      val byKeyMatchingAzp = Consumer.find(By(Consumer.key, azp.getOrElse("None")))
+      if (byKeyMatchingAzp.isDefined) {
+        val c = byKeyMatchingAzp.openOrThrowException("checked isDefined")
+        logger.info(s"getOrCreateConsumer says: MATCH on lookup 2 (by Consumer.key matching azp). Found pre-registered consumer: consumerId=${c.consumerId.get}, key=${c.key.get}, azp=${c.azp.get}, iss=${c.iss.get}")
+        // Transitional cleanup: before the duplicate-consumer fix, OAuth2/OIDC flows could auto-create
+        // consumers that now conflict with the pre-registered one we just found. Clear the stale consumer's
+        // azp/iss/sub so we can populate those fields on the pre-registered consumer without a unique
+        // constraint violation. This block can be removed once all environments have been cleaned up.
+        val conflicting = Consumer.find(By(Consumer.azp, azp.getOrElse("None")), By(Consumer.iss, iss.getOrElse("None")))
+        for (stale <- conflicting) {
+          if (stale.id.get != c.id.get) {
+            logger.info(s"getOrCreateConsumer says: Found CONFLICTING auto-created consumer holding the same (azp, iss). Clearing its azp/iss/sub to avoid unique constraint violation. Stale consumer: consumerId=${stale.consumerId.get}, key=${stale.key.get}, azp=${stale.azp.get}, iss=${stale.iss.get}, sub=${stale.sub.get}")
+            stale.azp(APIUtil.generateUUID())
+            stale.sub(APIUtil.generateUUID())
+            stale.saveMe()
+            logger.info(s"getOrCreateConsumer says: Cleared stale consumer. Now: consumerId=${stale.consumerId.get}, azp=${stale.azp.get}, sub=${stale.sub.get}")
+          }
+        }
+        // End of transitional cleanup block
+        logger.info(s"getOrCreateConsumer says: Updating azp/iss/sub on pre-registered consumer so future lookups also match by (azp, iss)...")
+        // Populate azp, iss, sub on the existing consumer so future lookups can also find it by (azp, iss)
+        for (found <- byKeyMatchingAzp) {
+          azp.foreach(v => found.azp(v))
+          iss.foreach(v => found.iss(v))
+          sub.foreach(v => found.sub(v))
+          found.saveMe()
+          logger.info(s"getOrCreateConsumer says: Updated pre-registered consumer. Now: consumerId=${found.consumerId.get}, key=${found.key.get}, azp=${found.azp.get}, iss=${found.iss.get}, sub=${found.sub.get}")
+        }
+        byKeyMatchingAzp
+      } else {
+        logger.info(s"getOrCreateConsumer says: MISS on lookup 2 (no consumer has key=${azp.getOrElse("None")}). Trying lookup 3 (by azp+iss pair)...")
+
+        // 3rd try: find by (azp, iss) pair issued by External Identity Provider
+        // The azp field in a JWT represents the Authorized Party (OAuth 2.0 / OpenID Connect client application).
+        // The pair (azp, iss) is a unique key in case of Client of an Identity Provider
+        val byAzpIss = Consumer.find(By(Consumer.azp, azp.getOrElse("None")), By(Consumer.iss, iss.getOrElse("None")))
+        if (byAzpIss.isDefined) {
+          val c = byAzpIss.openOrThrowException("checked isDefined")
+          logger.info(s"getOrCreateConsumer says: MATCH on lookup 3 (by azp+iss). Found auto-created consumer: consumerId=${c.consumerId.get}, key=${c.key.get}, azp=${c.azp.get}, iss=${c.iss.get}")
+          byAzpIss
+        } else {
+          logger.info(s"getOrCreateConsumer says: MISS on all 3 lookups. Will CREATE a new consumer. Searched: consumerId=${consumerId.getOrElse("None")}, key=${azp.getOrElse("None")}, (azp=${azp.getOrElse("None")}, iss=${iss.getOrElse("None")})")
+          Empty
         }
+      }
+    }
     consumer match {
       case Full(c) => Full(c)
       case Failure(msg, t, c) => Failure(msg, t, c)
       case ParamFailure(x,y,z,q) => ParamFailure(x,y,z,q)
       case Empty =>
         tryo {
           val c = Consumer.create
-          key match {
-            case Some(v) => c.key(v)
-            case None =>
-          }
-          secret match {
-            case Some(v) => c.secret(v)
-            case None =>
+          val actualKey = key.getOrElse(Helpers.randomString(40).toLowerCase)
+          val actualSecret = secret.getOrElse(Helpers.randomString(40).toLowerCase)
+          val actualConsumerId = consumerId.getOrElse {
+            azp match {
+              case Some(value) if APIUtil.checkIfStringIsUUID(value) => value
+              case Some(value) => s"${value}_${APIUtil.generateUUID()}"
+              case None => APIUtil.generateUUID()
+            }
           }
+          c.key(actualKey)
+          c.secret(actualSecret)
           aud match {
             case Some(v) => c.aud(v)
             case None =>
@@ -480,10 +532,7 @@ object MappedConsumersProvider extends ConsumersProvider with MdcLoggable {
             case Some(v) => c.logoUrl(v)
             case None =>
           }
-          consumerId match {
-            case Some(v) => c.consumerId(v)
-            case None =>
-          }
+          c.consumerId(actualConsumerId)
           val createdConsumer = c.saveMe()
           // In case we use Hydra ORY as Identity Provider we create corresponding client at Hydra side a well
           if(integrateWithHydra) createHydraClient(createdConsumer)

obp-api/src/main/scala/code/util/SecureLogging.scala

@@ -2,7 +2,7 @@ package code.util
 
 import code.api.util.APIUtil
 
-import java.util.regex.Pattern
+import java.util.regex.{Matcher, Pattern}
 import scala.collection.mutable
 
 /**
@@ -21,98 +21,109 @@ object SecureLogging {
   private def conditionalPattern(
                                   prop: String,
                                   defaultValue: Boolean = true
-                                )(pattern: => (Pattern, String)): Option[(Pattern, String)] = {
+                                )(pattern: => (Pattern, Matcher => String)): Option[(Pattern, Matcher => String)] = {
     if (APIUtil.getPropsAsBoolValue(prop, defaultValue)) Some(pattern) else None
   }
 
+  /** Helper to create a static replacement function from a replacement string */
+  private def staticReplacement(replacement: String): Matcher => String = _ => replacement
+
+  /** Helper to create a partial-mask replacement that shows first 3 and last 3 chars of group 2 */
+  private def partialMaskReplacement: Matcher => String = m => {
+    val prefix = m.group(1)
+    val value = m.group(2)
+    if (value.length > 6) s"${prefix}${value.take(3)}...${value.takeRight(3)}"
+    else s"${prefix}***"
+  }
+
   /**
    * Toggleable sensitive patterns.
    * Note: The sensitive keywords are defined in APIUtil.sensitiveKeywords.
    * When adding new categories here, also update that shared list.
    */
-  private lazy val sensitivePatterns: List[(Pattern, String)] = {
+  private lazy val sensitivePatterns: List[(Pattern, Matcher => String)] = {
     val patterns = Seq(
       // OAuth2 / API secrets
       conditionalPattern("securelogging_mask_secret") {
-        (Pattern.compile("(?i)(secret=)([^,\\s&]+)"), "$1***")
+        (Pattern.compile("(?i)(secret=)([^,\\s&]+)"), staticReplacement("$1***"))
       },
       conditionalPattern("securelogging_mask_client_secret") {
-        (Pattern.compile("(?i)(client_secret[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), "$1***")
+        (Pattern.compile("(?i)(client_secret[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), staticReplacement("$1***"))
       },
       conditionalPattern("securelogging_mask_client_secret") {
-        (Pattern.compile("(?i)(client_secret\\s*->\\s*)([^,\\s&\\)]+)"), "$1***")
+        (Pattern.compile("(?i)(client_secret\\s*->\\s*)([^,\\s&\\)]+)"), staticReplacement("$1***"))
       },
 
       // Authorization / Tokens
       conditionalPattern("securelogging_mask_authorization") {
-        (Pattern.compile("(?i)(Authorization:\\s*Bearer\\s+)([^\\s,&]+)"), "$1***")
+        (Pattern.compile("(?i)(Authorization:\\s*Bearer\\s+)([^\\s,&]+)"), staticReplacement("$1***"))
       },
       conditionalPattern("securelogging_mask_access_token") {
-        (Pattern.compile("(?i)(access_token[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), "$1***")
+        (Pattern.compile("(?i)(access_token[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), staticReplacement("$1***"))
       },
       conditionalPattern("securelogging_mask_access_token") {
-        (Pattern.compile("(?i)(access_token\\s*->\\s*)([^,\\s&\\)]+)"), "$1***")
+        (Pattern.compile("(?i)(access_token\\s*->\\s*)([^,\\s&\\)]+)"), staticReplacement("$1***"))
       },
       conditionalPattern("securelogging_mask_refresh_token") {
-        (Pattern.compile("(?i)(refresh_token[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), "$1***")
+        (Pattern.compile("(?i)(refresh_token[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), staticReplacement("$1***"))
       },
       conditionalPattern("securelogging_mask_refresh_token") {
-        (Pattern.compile("(?i)(refresh_token\\s*->\\s*)([^,\\s&\\)]+)"), "$1***")
+        (Pattern.compile("(?i)(refresh_token\\s*->\\s*)([^,\\s&\\)]+)"), staticReplacement("$1***"))
       },
       conditionalPattern("securelogging_mask_id_token") {
-        (Pattern.compile("(?i)(id_token[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), "$1***")
+        (Pattern.compile("(?i)(id_token[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), staticReplacement("$1***"))
       },
       conditionalPattern("securelogging_mask_id_token") {
-        (Pattern.compile("(?i)(id_token\\s*->\\s*)([^,\\s&\\)]+)"), "$1***")
+        (Pattern.compile("(?i)(id_token\\s*->\\s*)([^,\\s&\\)]+)"), staticReplacement("$1***"))
       },
       conditionalPattern("securelogging_mask_token") {
-        (Pattern.compile("(?i)(token[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), "$1***")
+        (Pattern.compile("(?i)(token[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), staticReplacement("$1***"))
       },
       conditionalPattern("securelogging_mask_token") {
-        (Pattern.compile("(?i)(token\\s*->\\s*)([^,\\s&\\)]+)"), "$1***")
+        (Pattern.compile("(?i)(token\\s*->\\s*)([^,\\s&\\)]+)"), staticReplacement("$1***"))
       },
 
       // Passwords
       conditionalPattern("securelogging_mask_password") {
-        (Pattern.compile("(?i)(password[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), "$1***")
+        (Pattern.compile("(?i)(password[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), staticReplacement("$1***"))
       },
       conditionalPattern("securelogging_mask_password") {
-        (Pattern.compile("(?i)(password\\s*->\\s*)([^,\\s&\\)]+)"), "$1***")
+        (Pattern.compile("(?i)(password\\s*->\\s*)([^,\\s&\\)]+)"), staticReplacement("$1***"))
       },
 
-      // API keys
+      // API keys - use partial masking to show first 3 and last 3 characters
       conditionalPattern("securelogging_mask_api_key") {
-        (Pattern.compile("(?i)(api_key[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), "$1***")
+        (Pattern.compile("(?i)(api_key[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), partialMaskReplacement)
       },
       conditionalPattern("securelogging_mask_api_key") {
-        (Pattern.compile("(?i)(api_key\\s*->\\s*)([^,\\s&\\)]+)"), "$1***")
+        (Pattern.compile("(?i)(api_key\\s*->\\s*)([^,\\s&\\)]+)"), partialMaskReplacement)
       },
       conditionalPattern("securelogging_mask_key") {
-        (Pattern.compile("(?i)(key[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), "$1***")
+        (Pattern.compile("(?i)(key[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), partialMaskReplacement)
       },
       conditionalPattern("securelogging_mask_key") {
-        (Pattern.compile("(?i)(key\\s*->\\s*)([^,\\s&\\)]+)"), "$1***")
+        (Pattern.compile("(?i)(key\\s*->\\s*)([^,\\s&\\)]+)"), partialMaskReplacement)
       },
       conditionalPattern("securelogging_mask_private_key") {
-        (Pattern.compile("(?i)(private_key[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), "$1***")
+        (Pattern.compile("(?i)(private_key[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+)"), staticReplacement("$1***"))
       },
       conditionalPattern("securelogging_mask_private_key") {
-        (Pattern.compile("(?i)(private_key\\s*->\\s*)([^,\\s&\\)]+)"), "$1***")
+        (Pattern.compile("(?i)(private_key\\s*->\\s*)([^,\\s&\\)]+)"), staticReplacement("$1***"))
       },
 
       // Database
       conditionalPattern("securelogging_mask_jdbc") {
-        (Pattern.compile("(?i)(jdbc:[^\\s]+://[^:]+:)([^@\\s]+)(@)"), "$1***$3")
+        (Pattern.compile("(?i)(jdbc:[^\\s]+://[^:]+:)([^@\\s]+)(@)"), staticReplacement("$1***$3"))
       },
 
       // Credit card
       conditionalPattern("securelogging_mask_credit_card") {
-        (Pattern.compile("\\b([0-9]{4})[\\s-]?([0-9]{4})[\\s-]?([0-9]{4})[\\s-]?([0-9]{3,7})\\b"), "$1-****-****-$4")
+        (Pattern.compile("\\b([0-9]{4})[\\s-]?([0-9]{4})[\\s-]?([0-9]{4})[\\s-]?([0-9]{3,7})\\b"), staticReplacement("$1-****-****-$4"))
       },
 
       // Email addresses
       conditionalPattern("securelogging_mask_email") {
-        (Pattern.compile("(?i)(email[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+@[^\"',\\s&]+)"), "$1***@***.***")
+        (Pattern.compile("(?i)(email[\"']?\\s*[:=]\\s*[\"']?)([^\"',\\s&]+@[^\"',\\s&]+)"), staticReplacement("$1***@***.***"))
       }
     )
 
@@ -129,8 +140,22 @@ object SecureLogging {
     val msgString = Option(msg).map(_.toString).getOrElse("")
     if (msgString.isEmpty) return msgString
 
-    sensitivePatterns.foldLeft(msgString) { case (acc, (pattern, replacement)) =>
-      pattern.matcher(acc).replaceAll(replacement)
+    sensitivePatterns.foldLeft(msgString) { case (acc, (pattern, replaceFn)) =>
+      val matcher = pattern.matcher(acc)
+      val sb = new StringBuffer()
+      while (matcher.find()) {
+        val replacement = replaceFn(matcher)
+        // If the function returns a string with $ references (static replacements),
+        // use appendReplacement which handles group references.
+        // Otherwise, quote the replacement to avoid $ interpretation.
+        if (replacement.contains("$1") || replacement.contains("$2") || replacement.contains("$3") || replacement.contains("$4")) {
+          matcher.appendReplacement(sb, replacement)
+        } else {
+          matcher.appendReplacement(sb, Matcher.quoteReplacement(replacement))
+        }
+      }
+      matcher.appendTail(sb)
+      sb.toString
     }
   }