Merge pull request #11 from BiobankLab/sync_fork_6

Sync fork 6

Author: kkochel

.github/integration/scripts/charts/dependencies.sh

@@ -9,7 +9,7 @@ random-string() {
 }
 
 if [ "$1" == "local" ]; then
-        if [ ! "$(command crypt4gh)" ]; then
+        if [ ! "$(command crypt4gh --version)" ]; then
                 echo "crypt4gh not installed, get it from here: https://github.com/neicnordic/crypt4gh/releases/latest"
                 exit 1
         elif [ "$(crypt4gh --version | cut -d ' ' -f1)" == "GA4GH" ]; then
@@ -18,10 +18,20 @@ if [ "$1" == "local" ]; then
                 exit 1
         fi
 
-        if [ ! "$(command yq)" ]; then
+        if [ ! "$(command yq --version)" ]; then
                 echo "yq not installed, get it from here: https://github.com/mikefarah/yq/releases/latest"
                 exit 1
         fi
+
+        if [ ! "$(command jq --version)" ]; then
+                echo "jq not installed"
+                exit 1
+        fi
+
+        if [ ! "$(command xxd --version 2>&1)" ]; then
+                echo "xxd not installed"
+                exit 1
+        fi
 else
         sudo curl --retry 100 -sL "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" -o /usr/bin/yq &&
                 sudo chmod +x /usr/bin/yq
@@ -104,4 +114,4 @@ yq -i '
 .releasetest.secrets.accessToken = strenv(TEST_TOKEN)
 ' "$values_file"
 
-kubectl create secret generic api-rbac --from-file=".github/integration/sda/rbac.json"
+kubectl create secret generic api-rbac --from-file=".github/integration/sda/rbac.json"

.github/integration/scripts/charts/deploy_charts.sh

@@ -36,6 +36,7 @@ if [ "$1" == "sda-mq" ]; then
         --set image.pullPolicy=IfNotPresent \
         --set global.adminPassword="$ADMINPASS" \
         --set global.adminUser=admin \
+        --set global.ingress.hostName=broker.127.0.0.1.nip.io \
         --set global.tls.enabled="$3" \
         --set global.tls.clusterIssuer=cert-issuer \
         --set persistence.enabled=false \

.github/integration/scripts/charts/values.yaml

@@ -1,12 +1,13 @@
 global:
   schemaType: "isolated"
   ingress:
-    deploy: false
+    deploy: true
     hostName:
-      api: pipeline-sda-svc-api
-      auth: pipeline-sda-svc-auth
-      download: pipeline-sda-svc-download
-      s3Inbox: pipeline-sda-svc-inbox
+      api: api.127.0.0.1.nip.io
+      auth: auth.127.0.0.1.nip.io
+      download: download.127.0.0.1.nip.io
+      s3Inbox: inbox.127.0.0.1.nip.io
+      syncapi: sync-api.127.0.0.1.nip.io
   log:
     level: "debug"
   tls:

.github/integration/scripts/make_certs.sh

@@ -34,6 +34,7 @@ openssl x509 -req -in "$out_dir/mq.csr" -days 1200 -CA "$out_dir/ca.crt" -CAkey
 
 # Create client certificate
 openssl req -config "$script_dir/ssl.cnf" -new -nodes -newkey rsa:4096 -keyout "$out_dir/client.key" -out "$out_dir/client.csr" -extensions client_cert -subj "/CN=admin"
+openssl pkcs8 -topk8 -inform PEM -outform DER -in "$out_dir/client.key" -out "$out_dir/client.der" -nocrypt
 openssl x509 -req -in "$out_dir/client.csr" -days 1200 -CA "$out_dir/ca.crt" -CAkey "$out_dir/ca-key.pem" -set_serial 01 -out "$out_dir/client.crt" -extensions client_cert -extfile "$script_dir/ssl.cnf"
 
 if [ -n "$KEYSTORE_PASSWORD" ]; then
@@ -68,7 +69,9 @@ chmod 600 /certs/*.key
 cp -p "$out_dir/ca.crt" /client_certs/ca.crt
 cp -p "$out_dir/client.crt" /client_certs/
 cp -p "$out_dir/client.key" /client_certs/
+cp -p "$out_dir/client.der" /client_certs/
 chmod 600 /client_certs/*.key
+chmod 644 /client_certs/*.der
 
 # needed if testing locally
 mkdir -p /temp/certs

.github/integration/scripts/make_sda_credentials.sh

@@ -29,19 +29,58 @@ done
 mkdir -p /shared/keys/pub
 if [ ! -f "/shared/keys/jwt.key" ]; then
     echo "creating jwt key"
-    openssl ecparam -genkey -name prime256v1 -noout -out /shared/keys/jwt.key
-    openssl ec -in /shared/keys/jwt.key -outform PEM -pubout >/shared/keys/pub/jwt.pub
+    cat << 'EOF' > /shared/keys/jwt.key
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDhuZjxPmOGUIW1
+LhxzKfxkN+1aTbvI5w+AptqT33X+bWuzfjvhEodiNz0bBfQgJJpQ3TZ8J1IZpM2F
+Tnzox+FGxKPe5T9Mgngzd4N6eByWVPXoNMk7IdmBXMdPZBFSyjMW4ba1MELCpiKV
+05de4J5opRDwmHmyMqYJxBk78e3iiYYixVk+j1Ku+yFl4d2R29y2+O9PlZegJloe
+8FGnKIGZApS/8t9iyCkXg8WbjSPzgYCTQKxn/E4lcGdTrAt/McKrWmAuppcr+rpP
++BInm3l5Zu/QiRSZcMb5O460ojP9eKnaUlDpGZv9CY5j4x4lq8vjU2kK77YXBO8I
+2oxse5a5AgMBAAECggEABbwSX6anHqVzECxQurhJWj51gELTT4JXSXxztygJNmKP
+RushGFHBMMSYf9RB5IMpjH5iQPs6wb4HHqjk0YEqfwLF6wbF+eqipSQXKghdKZCV
+AsY8io0MmpXB1omDSygp7h3j52yHdayE2muav+VTAPOYn5QwG0/gGgVqYrR9x7CM
+iTuyOIuGNO4Wlly4/5RhLtSo0pal9AgBvX4crtVEwN8tPgqPVo9w71bSROt9EVNI
+3cZiFFrrapYiifckIGiPGQYQUd5ej9Mq/77Fa0fv0pk0ONQV8HwstQ5HY2WwJWsn
+mccF9plVTzem7N/vo+T+hFRPUO9TZUao91mMV8iV5QKBgQD1nZbQW3NHdol0fXA8
+nw5JRkTLZx1zcZ5l36WVPkwCjJOyXQ2vWHm4lz7F81Rr8dQnMKLWMDKjrBT9Dbfs
+xYK2bYxENS1W/n+0jOIaX/792DY9tfX7vvHU9yGSdoJE5os6DGCHYInOD0xnRmnl
+3vS7gKv8miDwDzFsbjtDg6WfSwKBgQDrRLkmmfZCMcmLA02YSrErAlUseuyad7lY
+HEJApXKfn262iHELlQa2zOBZpJGXIcHsNf1XGpMeU5pH+ILKE4Y5qbclq+AzFCcZ
+nBFUfDeawmWdV5FJqNDd1L8Mb8aE+6q0Y5rNb3RL7A2ypH2ZeYKSGpHz3C7Rn5KW
+voWAXRWriwKBgQCH4bxK3x0ivxiCgtcyIojDzwVGRnDLqmMIVzeDHqjsjBs2BTcJ
+9/e3QK1w1BKzeWF2oPilaJrLY+tkqE9FxWtwQ6DjJ0xDIZ9DIuH/13X5t8EiWOWS
+devSdzpyje+58JW78pcArk7u2hXZ2OHDU5qvlRsRL6/jP3SHWWCeFFnviwKBgGov
+M02r0YygwfEfBYeFtp7Nx7lypZU2Eg4levWIdsp6f9KclEEA+u3IXD25XAiVMNw2
+pegJU3stioWPMSCZXUxrQAEdqOwE3XzehqfWBJaxxIEWQ7m2Gsb0PWIUlMnyeGJA
+Tl8IPboCiVAmk5WQVREyMsuYhf0Qg23MAZ8k5CHvAoGBAJm55NQZVKAEDGd4a21q
+TDcRddtPwwL2oP3qa0gbGk4YFRUCrX99hIejOTvQW1xf6vGxTd7E1QizvFse4yRz
+ZRKyXIc7DCcdzOnpMrSd1+aXwZtRHLSw0EDS6PWeJZdjJYHxl2YpAmMdURdcGTrH
+b6b/6vhU90+xL14CX7Awofp/
+-----END PRIVATE KEY-----
+EOF
+    cat << 'EOF' > /shared/keys/pub/jwt.pub
+-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4bmY8T5jhlCFtS4ccyn8
+ZDftWk27yOcPgKbak991/m1rs3474RKHYjc9GwX0ICSaUN02fCdSGaTNhU586Mfh
+RsSj3uU/TIJ4M3eDengcllT16DTJOyHZgVzHT2QRUsozFuG2tTBCwqYildOXXuCe
+aKUQ8Jh5sjKmCcQZO/Ht4omGIsVZPo9SrvshZeHdkdvctvjvT5WXoCZaHvBRpyiB
+mQKUv/LfYsgpF4PFm40j84GAk0CsZ/xOJXBnU6wLfzHCq1pgLqaXK/q6T/gSJ5t5
+eWbv0IkUmXDG+TuOtKIz/Xip2lJQ6Rmb/QmOY+MeJavL41NpCu+2FwTvCNqMbHuW
+uQIDAQAB
+-----END PUBLIC KEY-----
+EOF
     chmod 644 /shared/keys/pub/jwt.pub /shared/keys/jwt.key
 fi
 
 echo "creating token"
-token="$(python /scripts/sign_jwt.py)"
+python /scripts/sign_jwt.py testu@lifescience-ri.eu  > "/shared/token"
 
 cat >/shared/s3cfg <<EOD
 [default]
-access_key=test_dummy.org
-secret_key=test_dummy.org
-access_token=$token
+access_key=test@dummy.org
+secret_key=test@dummy.org
+access_token="$(python /scripts/sign_jwt.py test@dummy.org)"
 check_ssl_certificate = False
 check_ssl_hostname = False
 encoding = UTF-8

.github/integration/scripts/sign_jwt.py

@@ -1,26 +1,27 @@
 from datetime import date, timedelta
 from joserfc import jwt
-from joserfc.jwk import ECKey
+from joserfc.jwk import RSAKey
 from pathlib import Path
+import sys
 
 p = Path('/shared/keys/jwt.key')
 raw = p.read_text()
-key = ECKey.import_key(raw)
+key = RSAKey.import_key(raw)
 iat = date.today() - timedelta(days=1)
 exp = date.today() + timedelta(days=1)
 
 header = {
-    'alg': 'ES256',
-    'kid': key.thumbprint(),
+    'alg': 'RS256',
+    'kid': 'rsa1',
     'typ': 'JWT'
 }
 
 payload = {
     'aud': 'XC56EL11xx',
     'exp': exp.strftime('%s'),
     'iat': iat.strftime('%s'),
-    'iss': 'http://oidc',
-    'sub': 'test@dummy.org'
+    'iss': 'http://localhost',
+    'sub': sys.argv[1]
 }
 
 token = jwt.encode(header, payload, key)

.github/integration/sda-doa-posix-outbox.yml

@@ -0,0 +1,158 @@
+services:
+  certfixer:
+    command:
+      - /bin/sh
+      - /scripts/make_certs.sh
+    container_name: certfixer
+    image: alpine:latest
+    volumes:
+      - ./scripts:/scripts
+      - certs:/certs
+      - client_certs:/client_certs
+      - /tmp:/temp
+
+  postgres:
+    build:
+      context: ../../postgresql
+    container_name: postgres
+    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres
+    depends_on:
+      certfixer:
+        condition: service_completed_successfully
+    ports:
+      - 5432:5432
+    environment:
+      - LEGA_IN_PASSWORD=password
+      - LEGA_OUT_PASSWORD=password
+      - POSTGRES_PASSWORD=rootpasswd
+      - POSTGRES_SERVER_CACERT=/certs/ca.crt
+      - POSTGRES_SERVER_CERT=/certs/db.crt
+      - POSTGRES_SERVER_KEY=/certs/db.key
+
+    healthcheck:
+      test: [ "CMD", "pg_isready", "-h", "localhost", "-U", "lega_out" ]
+      interval: 5s
+      timeout: 20s
+      retries: 3
+    volumes:
+      - certs:/certs
+
+  mockauth:
+    container_name: mockauth
+    image: python:3.11-slim
+    ports:
+      - 8000:8000
+    environment:
+      - ISSUER_URL=http://mockauth:8000
+    volumes:
+      - ../../sda-doa/test/mock_auth.py:/mock_auth.py
+      - client_certs:/client_certs
+    command: >
+      sh -c "pip install --upgrade pip && pip install aiohttp Authlib && python -u /mock_auth.py 0.0.0.0 8000"
+
+  rabbitmq:
+    build:
+      context: ../../rabbitmq
+    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq
+    container_name: rabbitmq
+    depends_on:
+      certfixer:
+        condition: service_completed_successfully
+    environment:
+      - RABBITMQ_SERVER_CACERT=/etc/rabbitmq/ssl/ca.crt
+      - RABBITMQ_SERVER_CERT=/etc/rabbitmq/ssl/mq.crt
+      - RABBITMQ_SERVER_KEY=/etc/rabbitmq/ssl/mq.key
+      - RABBITMQ_SERVER_VERIFY=verify_none
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "bash",
+          "-c",
+          "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_local_alarms",
+        ]
+      interval: 5s
+      timeout: 20s
+      retries: 3
+    restart: always
+    volumes:
+      - certs:/etc/rabbitmq/ssl/
+      - rabbitmq_data:/var/lib/rabbitmq
+    ports:
+      - 5671:5671
+      - 25671:15671
+
+  doa:
+    container_name: doa
+    build:
+      context: ../../sda-doa
+    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-doa
+    depends_on:
+      postgres:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+      certfixer:
+        condition: service_completed_successfully
+    ports:
+      - "8080:8080"
+    environment:
+      - SSL_MODE=verify-ca
+      - POSTGRES_USER=postgres
+      - POSTGRES_DB=sda
+      - POSTGRES_PASSWORD=rootpasswd
+      - DB_INSTANCE=postgres
+      - SSL_ENABLED=false
+      - BROKER_HOST=rabbitmq
+      - BROKER_VALIDATE=false
+      - OPENID_CONFIGURATION_URL=http://mockauth:8000/openid-configuration
+      - USERINFO_ENDPOINT_URL=http://mockauth:8000/userinfo
+      - CRYPT4GH_PRIVATE_KEY_PATH=test/crypt4gh.sec.pem
+      - CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH=test/crypt4gh.pass
+      - OUTBOX_TYPE=POSIX
+      - OUTBOX_LOCATION=/outbox/%s/files/
+      - SSL_ENABLED=false
+      - ROOT_CERT_PATH=/certs/ca.crt
+      - CERT_PATH=/certs/client.crt
+      - CERT_KEY=/certs/client.der
+      - BROKER_USERNAME=guest
+
+    volumes:
+      - client_certs:/certs
+      - ../../sda-doa/test/body.enc:/test/body.enc
+      - test_file:/outbox
+      - encryption_files:/test
+
+  integration_test:
+    container_name: integration_test
+    image: maven:3.9.9-eclipse-temurin-21
+    profiles: [test]
+    volumes:
+      - ../../sda-doa/src:/sda-doa/src
+      - ../../sda-doa/pom.xml:/sda-doa/pom.xml
+      - ../../sda-doa/settings.xml:/root/.m2/settings.xml
+      - test_file:/sda-doa/outbox
+      - ./tests:/tests
+      - encryption_files:/test
+      - client_certs:/certs
+
+    depends_on:
+      - doa
+      - mockauth
+    environment:
+      - OUTBOX_TYPE=POSIX
+      - DOA_URL=http://doa:8080
+      - MOCKAUTH_URL=http://mockauth:8000
+      - MINIO_HOST=outbox
+
+    command:
+      - "/bin/sh"
+      - "/tests/run_scripts.sh"
+      - "/tests/doa"
+
+volumes:
+  certs:
+  client_certs:
+  rabbitmq_data:
+  test_file:
+  encryption_files: