Merge pull request #2742 from simonredfern/develop

ConsentItem table so we don't have to extract JWT for bank_id endpoints

Author: simonredfern

flushall_build_and_run.sh

@@ -111,8 +111,8 @@ tail -n 3 -f build.log &
 TAIL_PID=$!
 mvn -pl obp-api -am clean package -DskipTests=true -Dmaven.test.skip=true -T 4 > build.log 2>&1
 BUILD_EXIT=$?
-kill $TAIL_PID 2>/dev/null
-wait $TAIL_PID 2>/dev/null
+kill $TAIL_PID 2>/dev/null || true
+wait $TAIL_PID 2>/dev/null || true
 
 if [ $BUILD_EXIT -ne 0 ]; then
     echo ""

obp-api/src/main/scala/bootstrap/liftweb/Boot.scala

@@ -63,7 +63,7 @@ import code.branches.MappedBranch
 import code.cardattribute.MappedCardAttribute
 import code.cards.{MappedPhysicalCard, PinReset}
 import code.connectormethod.ConnectorMethod
-import code.consent.{ConsentRequest, MappedConsent}
+import code.consent.{ConsentItem, ConsentRequest, MappedConsent}
 import code.consumer.Consumers
 import code.model.Consumer
 import code.context.{MappedConsentAuthContext, MappedUserAuthContext, MappedUserAuthContextUpdate}
@@ -1120,6 +1120,7 @@ object ToSchemify {
     MappedCustomerIdMapping,
     MappedProductAttribute,
     MappedConsent,
+    ConsentItem,
     ConsentRequest,
     MigrationScriptLog,
     MethodRouting,

obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala

@@ -4601,9 +4601,10 @@ object SwaggerDefinitionsJSON {
     last_action_date = dateExample.value,
     last_usage_date = dateTimeExample.value,
     jwt = jwtExample.value,
-    jwt_payload = Some(consentJWT),
+    jwt_payload = """{"createdByUserId":"user-id","sub":"subject","iss":"issuer","aud":"audience","jti":"jwt-id","iat":1611749820,"nbf":1611749820,"exp":1611753420,"request_headers":[],"name":null,"email":null,"entitlements":[],"views":[],"access":null}""",
     api_standard = "Berlin Group",
     api_version = "v1.3",
+    jwt_expires_at = dateTimeExample.value,
   )
 
   lazy val consentsInfoJsonV510 = ConsentsInfoJsonV510(

obp-api/src/main/scala/code/api/util/APIUtil.scala

@@ -1271,19 +1271,13 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
       roleName <- getHttpParamValuesByName(httpParams, "role_name")
       httpStatusCode <- getHttpParamValuesByName(httpParams, "http_status_code")
     }yield{
-      /**
-       * sortBy is currently disabled as it would open up a security hole:
-       *
-       * sortBy as currently implemented will take in a parameter that searches on the mongo field names. The issue here
-       * is that it will sort on the true value, and not the moderated output. So if a view is supposed to return an alias name
-       * rather than the true value, but someone uses sortBy on the other bank account name/holder, not only will the returned data
-       * have the wrong order, but information about the true account holder name will be exposed due to its position in the sorted order
-       *
-       * This applies to all fields that can have their data concealed... which in theory will eventually be most/all
-       *
-       */
-      //val sortBy = json.header("obp_sort_by")
-      val ordering = OBPOrdering(None, sortDirection)
+      // Extract the sort field name from the sort_by query param (e.g. "url", "date").
+      // OBPOrdering expects Option[String], but sortBy is an OBPQueryParam.
+      val sortField = sortBy match {
+        case OBPSortBy(v) => Some(v)
+        case _ => None
+      }
+      val ordering = OBPOrdering(sortField, sortDirection)
       //This guarantee the order
       List(limit, offset, ordering, sortBy, fromDate, toDate,
         anon, status, consumerId, azp, iss, consentId, userId, providerProviderId, url, appName, implementedByPartialFunction, implementedInVersion,

obp-api/src/main/scala/code/api/util/Glossary.scala

@@ -5111,6 +5111,20 @@ object Glossary extends MdcLoggable  {
 				 |
 """)
 
+	glossaryItems += GlossaryItem(
+		title = "SDKs",
+		description =
+			s"""
+				 |# SDKs
+				 |
+				 |OBP SDKs (Software Development Kits) are client libraries that make it easier to interact with the OBP API from various programming languages.
+				 |
+				 |SDKs are available for multiple languages including Python, Java, Scala, PHP, C#, Javascript and more.
+				 |
+				 |For more information see [OBP SDKs on GitHub](https://github.com/OpenBankProject/OBP-SDKs).
+				 |
+""")
+
 	///////////////////////////////////////////////////////////////////
 	// NOTE! Some glossary items are generated in ExampleValue.scala
 //////////////////////////////////////////////////////////////////

obp-api/src/main/scala/code/api/util/migration/Migration.scala

@@ -113,6 +113,7 @@ object Migration extends MdcLoggable {
       addMetricView(startedBeforeSchemifier)
       addConsentView(startedBeforeSchemifier)
       updateConsentViewAddJwtPayload(startedBeforeSchemifier)
+      updateConsentViewAddJwtExpiresAt(startedBeforeSchemifier)
       updateAccountAccessWithViewsViewUnionAll(startedBeforeSchemifier)
     }
 
@@ -613,6 +614,18 @@ object Migration extends MdcLoggable {
       }
     }
 
+    private def updateConsentViewAddJwtExpiresAt(startedBeforeSchemifier: Boolean): Boolean = {
+      if(startedBeforeSchemifier == true) {
+        logger.warn(s"Migration.database.updateConsentViewAddJwtExpiresAt(true) cannot be run before Schemifier.")
+        true
+      } else {
+        val name = nameOf(updateConsentViewAddJwtExpiresAt(startedBeforeSchemifier))
+        runOnce(name) {
+          MigrationOfConsentView.addConsentView(name)
+        }
+      }
+    }
+
     private def addAccountAccessWithViewsView(startedBeforeSchemifier: Boolean): Boolean = {
       if(startedBeforeSchemifier == true) {
         logger.warn(s"Migration.database.addAccountAccessWithViewsView(true) cannot be run before Schemifier.")

obp-api/src/main/scala/code/api/util/migration/MigrationOfConsentView.scala

@@ -37,7 +37,8 @@ object MigrationOfConsentView {
                     |    mnote                              AS note,
                     |    mfrequencyperday                   AS frequency_per_day,
                     |    musessofartodaycounter             AS uses_so_far_today_counter,
-                    |    mjsonwebtokenpayload               AS jwt_payload
+                    |    mjsonwebtokenpayload               AS jwt_payload,
+                    |    jwt_expires_at                     AS jwt_expires_at
                     |FROM mappedconsent;
                     |""".stripMargin
               case _ =>
@@ -60,7 +61,8 @@ object MigrationOfConsentView {
                     |    mnote                              AS note,
                     |    mfrequencyperday                   AS frequency_per_day,
                     |    musessofartodaycounter             AS uses_so_far_today_counter,
-                    |    mjsonwebtokenpayload               AS jwt_payload
+                    |    mjsonwebtokenpayload               AS jwt_payload,
+                    |    jwt_expires_at                     AS jwt_expires_at
                     |FROM mappedconsent;
                     |""".stripMargin
             }

obp-api/src/main/scala/code/api/v3_1_0/APIMethods310.scala

@@ -25,7 +25,7 @@ import code.api.v3_0_0.{CreateViewJsonV300, JSONFactory300}
 import code.api.v3_1_0.JSONFactory310._
 import code.bankconnectors.rest.RestConnector_vMar2019
 import code.bankconnectors.{Connector, LocalMappedConnector}
-import code.consent.{ConsentStatus, Consents, MappedConsent}
+import code.consent.{ConsentStatus, Consents, DoobieConsentQueries, MappedConsent}
 import code.consumer.Consumers
 import code.entitlement.Entitlement
 import code.loginattempts.LoginAttempt
@@ -3726,6 +3726,10 @@ trait APIMethods310 {
         |
         |${userAuthenticationMessage(true)}
         |
+        |1 limit (for pagination: defaults to 50)  eg:limit=200
+        |
+        |2 offset (for pagination: zero index, defaults to 0) eg: offset=10
+        |
       """.stripMargin,
       EmptyBody,
       consentsJsonV310,
@@ -3739,12 +3743,32 @@ trait APIMethods310 {
     lazy val getConsents: OBPEndpoint = {
       case "banks" :: BankId(bankId) :: "my" :: "consents" :: Nil JsonGet _ => {
         cc => implicit val ec = EndpointContext(Some(cc))
+          val url = cc.url
+          val limitParam = APIUtil.getHttpRequestUrlParam(url, "limit") match {
+            case s if s.nonEmpty => scala.util.Try(s.toInt).getOrElse(50)
+            case _ => 50
+          }
+          val offsetParam = APIUtil.getHttpRequestUrlParam(url, "offset") match {
+            case s if s.nonEmpty => scala.util.Try(s.toInt).getOrElse(0)
+            case _ => 0
+          }
           for {
             (Full(user), callContext) <- authenticatedAccess(cc)
             (_, callContext) <- NewStyle.function.getBank(bankId, callContext)
-            consents <- Future(Consents.consentProvider.vend.getConsentsByUser(user.userId))
+            rows <- Future {
+              DoobieConsentQueries.getConsentsByUserAndBank(
+                userId = user.userId,
+                bankId = bankId.value,
+                status = None,
+                limit = limitParam,
+                offset = offsetParam,
+                sortField = "created_date",
+                sortDirection = "desc"
+              )
+            }
           } yield {
-            (JSONFactory310.createConsentsJsonV310(consents), HttpCode.`200`(callContext))
+            val consents = rows.map(r => ConsentJsonV310(r.consentId, r.jwt.getOrElse(""), r.status))
+            (ConsentsJsonV310(consents), HttpCode.`200`(callContext))
           }
       }
     }

obp-api/src/main/scala/code/api/v4_0_0/APIMethods400.scala

@@ -42,7 +42,7 @@ import code.authtypevalidation.JsonAuthTypeValidation
 import code.bankconnectors.LocalMappedConnectorInternal._
 import code.bankconnectors.{Connector, DynamicConnector, InternalConnector, LocalMappedConnectorInternal}
 import code.connectormethod.{JsonConnectorMethod, JsonConnectorMethodMethodBody}
-import code.consent.{ConsentStatus, Consents}
+import code.consent.{ConsentStatus, Consents, DoobieConsentQueries}
 import code.dynamicEntity.DynamicEntityCommons
 import code.dynamicMessageDoc.JsonDynamicMessageDoc
 import code.dynamicResourceDoc.JsonDynamicResourceDoc
@@ -11479,6 +11479,10 @@ trait APIMethods400 extends MdcLoggable {
          |
          |${userAuthenticationMessage(true)}
          |
+         |1 limit (for pagination: defaults to 50)  eg:limit=200
+         |
+         |2 offset (for pagination: zero index, defaults to 0) eg: offset=10
+         |
       """.stripMargin,
       EmptyBody,
       consentsJsonV400,
@@ -11494,19 +11498,33 @@ trait APIMethods400 extends MdcLoggable {
       case "banks" :: BankId(bankId) :: "my" :: "consents" :: Nil JsonGet _ => {
         cc =>
           implicit val ec = EndpointContext(Some(cc))
+          val url = cc.url
+          val limitParam = getHttpRequestUrlParam(url, "limit") match {
+            case s if s.nonEmpty => scala.util.Try(s.toInt).getOrElse(50)
+            case _ => 50
+          }
+          val offsetParam = getHttpRequestUrlParam(url, "offset") match {
+            case s if s.nonEmpty => scala.util.Try(s.toInt).getOrElse(0)
+            case _ => 0
+          }
           for {
-            consents <- Future {
-              Consents.consentProvider.vend
-                .getConsentsByUser(cc.userId)
-                .sortBy(i => (i.creationDateTime, i.apiStandard))
-                .reverse
+            rows <- Future {
+              DoobieConsentQueries.getConsentsByUserAndBank(
+                userId = cc.userId,
+                bankId = bankId.value,
+                status = None,
+                limit = limitParam,
+                offset = offsetParam,
+                sortField = "created_date",
+                sortDirection = "desc"
+              )
             }
           } yield {
-            val consentsOfBank = Consent.filterByBankId(consents, bankId)
-            (
-              JSONFactory400.createConsentsJsonV400(consentsOfBank),
-              HttpCode.`200`(cc)
-            )
+            val consents = rows.map(r => ConsentJsonV400(
+              r.consentId, r.jwt.getOrElse(""), r.status,
+              r.apiStandard.getOrElse(""), r.apiVersion.getOrElse("")
+            ))
+            (ConsentsJsonV400(consents), HttpCode.`200`(cc))
           }
       }
     }
@@ -12109,6 +12127,10 @@ trait APIMethods400 extends MdcLoggable {
          |
          |${userAuthenticationMessage(true)}
          |
+         |1 limit (for pagination: defaults to 50)  eg:limit=200
+         |
+         |2 offset (for pagination: zero index, defaults to 0) eg: offset=10
+         |
          |""".stripMargin,
       EmptyBody,
       apiCollectionsJson400,
@@ -12122,12 +12144,22 @@ trait APIMethods400 extends MdcLoggable {
     lazy val getMyApiCollections: OBPEndpoint = {
       case "my" :: "api-collections" :: Nil JsonGet _ => { cc =>
         implicit val ec = EndpointContext(Some(cc))
+        val url = cc.url
+        val limitParam = getHttpRequestUrlParam(url, "limit") match {
+          case s if s.nonEmpty => scala.util.Try(s.toInt).getOrElse(50)
+          case _ => 50
+        }
+        val offsetParam = getHttpRequestUrlParam(url, "offset") match {
+          case s if s.nonEmpty => scala.util.Try(s.toInt).getOrElse(0)
+          case _ => 0
+        }
         for {
           (apiCollections, callContext) <- NewStyle.function
             .getApiCollectionsByUserId(cc.userId, Some(cc))
         } yield {
+          val paginated = apiCollections.drop(offsetParam).take(limitParam)
           (
-            JSONFactory400.createApiCollectionsJsonV400(apiCollections),
+            JSONFactory400.createApiCollectionsJsonV400(paginated),
             HttpCode.`200`(callContext)
           )
         }