Merge pull request #13 from BiobankLab/sync-fork-7

Sync fork 7

Author: kkochel

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @neicnordic/sensitive-data-development-collaboration

.github/integration/scripts/charts/cleanup.sh

@@ -16,4 +16,6 @@ helm uninstall broker || true
 helm uninstall postgres || true
 
 kubectl delete secrets api-rbac broker-sda-mq-certs broker-sda-mq-test-certs postgres-sda-db-certs postgres-sda-db-test-certs || true
-rm /tmp/values.yaml /tmp/c4gh.pub.pem /tmp/c4gh.sec.pem /tmp/jwt.key /tmp/jwt.pub
+kubectl delete deployment.apps/cega-nss || true
+kubectl delete cm cega-nss || true
+rm /tmp/values.yaml /tmp/c4gh.pub.pem /tmp/c4gh.sec.pem /tmp/jwt.key /tmp/jwt.pub /tmp/users.json || true

.github/integration/scripts/charts/dependencies.sh

@@ -2,7 +2,7 @@
 set -ex
 
 YQ_VERSION="v4.20.1"
-C4GH_VERSION="$(curl --retry 100 -sL https://api.github.com/repos/neicnordic/crypt4gh/releases/latest | jq -r '.name')"
+C4GH_VERSION="v1.14.0"
 
 random-string() {
         head -c 32 /dev/urandom | base64 -w0 | tr -d '/+' | fold -w 32 | head -n 1
@@ -99,13 +99,17 @@ if [ "$1" == "local" ]; then
 fi
 
 ## update values file with all credentials
+if [ "$2" == "federated" ]; then
+        yq -i '.global.schemaType = federated' "$values_file"
+fi
+
 yq -i '
 .global.archive.s3AccessKey = strenv(MINIO_ACCESS) |
 .global.archive.s3SecretKey = strenv(MINIO_SECRET) |
 .global.backupArchive.s3AccessKey = strenv(MINIO_ACCESS) |
 .global.backupArchive.s3SecretKey = strenv(MINIO_SECRET) |
 .global.broker.password = strenv(MQPASSWORD) |
-.global.c4gh.passphrase = strenv(C4GHPASSPHRASE) |
+.global.c4gh.privateKeys[0].passphrase = strenv(C4GHPASSPHRASE) |
 .global.db.password = strenv(PGPASSWORD) |
 .global.inbox.s3AccessKey = strenv(MINIO_ACCESS) |
 .global.inbox.s3SecretKey = strenv(MINIO_SECRET) |
@@ -115,3 +119,17 @@ yq -i '
 ' "$values_file"
 
 kubectl create secret generic api-rbac --from-file=".github/integration/sda/rbac.json"
+
+cat >/tmp/users.json <<EOD
+[
+    {
+        "username": "dummy@example.com",
+        "uid": 1,
+        "passwordHash": "\$2b\$12\$1gyKIjBc9/cT0MYkXX24xe1LjEUjNwgL4rEk8fDoO.vDQZzWkqrn.",
+        "gecos": "dummy user",
+        "sshPublicKey": [],
+        "enabled": null
+    }
+]
+EOD
+kubectl create configmap cega-nss --from-file=".github/integration/sda/users.py" --from-file="/tmp/users.json"

.github/integration/scripts/charts/dependencies.yaml

@@ -91,6 +91,14 @@ spec:
           requests:
             cpu: 100m
             memory: 128Mi
+        volumeMounts:
+          - mountPath: /shared/keys
+            name: jwt
+      volumes:
+        - name: jwt
+          secret:
+            defaultMode: 288
+            secretName: jwk
 ---
 apiVersion: v1
 kind: Service
@@ -140,4 +148,63 @@ spec:
   resources:
     requests:
       storage: 1G
-  storageClassName: local-path
+  storageClassName: local-path
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cega-nss
+spec:
+  selector:
+    matchLabels:
+      app: cega-nss
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: cega-nss
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsGroup: 1000
+        runAsUser: 1000
+        fsGroup: 1000
+      containers:
+      - name: cega-nss
+        image: egarchive/lega-base:release.v0.2.0
+        command: ["python", "/cega/users.py", "0.0.0.0", "8443", "/cega/users.json"]
+        env:
+          - name: CEGA_USERS_PASSWORD
+            value: test
+          - name: CEGA_USERS_USER
+            value: test
+        ports:
+        - containerPort: 8443
+        resources:
+          limits:
+            cpu: 250m
+            memory: 256Mi
+          requests:
+            cpu: 100m
+            memory: 128Mi
+        volumeMounts:
+          - mountPath: /cega
+            name: app
+      volumes:
+        - name: app
+          configMap:
+            name: cega-nss
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: cega-nss
+  labels:
+    app: cega-nss
+spec:
+  type: ClusterIP
+  ports:
+  - port: 8443
+    targetPort: 8443
+  selector:
+    app: cega-nss

.github/integration/scripts/charts/deploy_charts.sh

@@ -7,8 +7,14 @@ if [ -z "$2" ]; then
 fi
 
 MQ_PORT=5672
+PROTOCOL=http
+SCHEME=HTTP
+GRPC_PORT=50051
 if [ "$3" == "true" ]; then
     MQ_PORT=5671
+    PROTOCOL=https
+    SCHEME=HTTPS
+    GRPC_PORT=50444
 fi
 
 dir=".github/integration/scripts/charts"
@@ -42,6 +48,10 @@ if [ "$1" == "sda-mq" ]; then
         --set persistence.enabled=false \
         --set resources=null \
         --wait
+
+    if [ "$4" == "federated" ]; then
+        curl -kL -u "admin:$ADMINPASS" -X PUT "$PROTOCOL://broker.127.0.0.1.nip.io/api/queues/sda/from_cega"
+    fi
 fi
 
 if [ "$1" == "sda-svc" ]; then
@@ -52,6 +62,7 @@ if [ "$1" == "sda-svc" ]; then
         sync_api_user=user
     fi
     helm install pipeline charts/sda-svc \
+        --set global.schemaType="$5" \
         --set image.tag="PR$2" \
         --set image.pullPolicy=IfNotPresent \
         --set global.tls.enabled="$3" \
@@ -62,6 +73,12 @@ if [ "$1" == "sda-svc" ]; then
         --set global.sync.api.password="$sync_api_pass" \
         --set global.sync.api.user="$sync_api_user" \
         --set global.sync.remote.host="$sync_host" \
+        --set api.readinessProbe.httpGet.scheme="$SCHEME" \
+        --set auth.readinessProbe.httpGet.scheme="$SCHEME" \
+        --set download.readinessProbe.httpGet.scheme="$SCHEME" \
+        --set s3Inbox.readinessProbe.httpGet.scheme="$SCHEME" \
+        --set syncAPI.readinessProbe.httpGet.scheme="$SCHEME" \
+        --set reencrypt.readinessProbe.grpc.port="$GRPC_PORT" \
         -f "$dir/values.yaml" \
         --wait
 fi

.github/integration/scripts/charts/k3d.sh

@@ -13,9 +13,11 @@ curl --retry 100 -sLO https://dl.k8s.io/release/"$k8s"/bin/linux/amd64/kubectl
 chmod +x ./kubectl
 sudo mv ./kubectl /usr/local/bin/kubectl
 
-k3d cluster create sda --image=rancher/k3s:"$k8s"-k3s1 --wait --timeout 10m
+k3d cluster create sda --image=rancher/k3s:"$k8s"-k3s1 --wait --timeout 10m --k3s-arg "--disable=traefik@server:0" --port "80:80@loadbalancer" --port "443:443@loadbalancer"
 k3d kubeconfig merge sda --kubeconfig-switch-context
 mkdir -p ~/.kube/ && cp ~/.config/kubeconfig-sda.yaml ~/.kube/config
 
 docker build -t ghcr.io/neicnordic/sensitive-data-archive:oidc -f .github/integration/scripts/charts/Dockerfile .
-k3d image import ghcr.io/neicnordic/sensitive-data-archive:oidc -c sda
+k3d image import ghcr.io/neicnordic/sensitive-data-archive:oidc -c sda
+
+helm upgrade --install ingress-nginx ingress-nginx --repo https://kubernetes.github.io/ingress-nginx --namespace ingress-nginx --create-namespace --version 4.11.6

.github/integration/scripts/charts/values.yaml

@@ -25,6 +25,7 @@ global:
     s3Url: "http://minio.minio"
     s3Bucket: "archive"
     s3Port: 9000
+    s3ReadyPath: "/minio/health/ready"
     existingClaim: archive-pvc
   backupArchive:
     storageType: "s3"
@@ -33,6 +34,7 @@ global:
     s3Url: "http://minio.minio"
     s3Bucket: "backup"
     s3Port: 9000
+    s3ReadyPath: "/minio/health/ready"
     existingClaim: backup-pvc
   auth:
     jwtSecret: jwk
@@ -49,13 +51,15 @@ global:
     username: "admin"
     password: PLACEHOLDER_VALUE
   cega:
+    host: https://cega-nss:8443/username
     password: PLACEHOLDER_VALUE
     user: PLACEHOLDER_VALUE
   c4gh:
     secretName: c4gh
-    keyFile: c4gh.sec.pem
-    publicFile: c4gh.pub.pem
-    passphrase: PLACEHOLDER_VALUE
+    publicKey: c4gh.pub.pem
+    privateKeys:
+      - keyName: c4gh.sec.pem
+        passphrase: PLACEHOLDER_VALUE
     syncPubKey: c4gh.pub.pem
   db:
     host: "postgres-sda-db"
@@ -90,8 +94,8 @@ global:
     port: 50443
   sync:
     api:
-      password: ""
-      user: ""
+      password: "apiuser"
+      user: "apipass"
     brokerQueue: "mapping_stream"
     centerPrefix: "SYNC"
     destination:
@@ -104,7 +108,7 @@ global:
       bucket: "sync"
       region: "us-east-1"
     remote:
-      host: ""
+      host: "http://remote-sync"
       port: "8080"
       password: "pass"
       user: "user"
@@ -127,7 +131,7 @@ finalize:
 ingest:
   resources: null
 intercept:
-  deploy: false
+  resources: null
 mapper:
   resources: null
 releasetest:

.github/integration/scripts/make_sda_credentials.sh

@@ -111,6 +111,11 @@ if [ ! -f "/shared/c4gh1.sec.pem" ]; then
     /shared/crypt4gh generate -n /shared/c4gh1 -p c4ghpass
 fi
 
+if [ ! -f "/shared/client.sec.pem" ]; then # client key for re-encryption
+    echo "creating client crypth4gh key"
+    /shared/crypt4gh generate -n /shared/client -p c4ghpass
+fi
+
 if [ ! -f "/shared/sync.sec.pem" ]; then
     echo "creating sync crypth4gh key"
     /shared/crypt4gh generate -n /shared/sync -p syncPass

.github/integration/sda-doa-posix-outbox.yml

@@ -15,7 +15,7 @@ services:
     build:
       context: ../../postgresql
     container_name: postgres
-    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres
+    image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-postgres
     depends_on:
       certfixer:
         condition: service_completed_successfully
@@ -53,7 +53,7 @@ services:
   rabbitmq:
     build:
       context: ../../rabbitmq
-    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq
+    image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq
     container_name: rabbitmq
     depends_on:
       certfixer:
@@ -86,7 +86,7 @@ services:
     container_name: doa
     build:
       context: ../../sda-doa
-    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-doa
+    image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-doa
     depends_on:
       postgres:
         condition: service_healthy
@@ -130,7 +130,6 @@ services:
     volumes:
       - ../../sda-doa/src:/sda-doa/src
       - ../../sda-doa/pom.xml:/sda-doa/pom.xml
-      - ../../sda-doa/settings.xml:/root/.m2/settings.xml
       - test_file:/sda-doa/outbox
       - ./tests:/tests
       - encryption_files:/test

.github/integration/sda-doa-s3-outbox.yml

@@ -15,7 +15,7 @@ services:
     build:
       context: ../../postgresql
     container_name: postgres
-    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres
+    image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-postgres
     depends_on:
       certfixer:
         condition: service_completed_successfully
@@ -53,7 +53,7 @@ services:
   rabbitmq:
     build:
       context: ../../rabbitmq
-    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq
+    image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq
     container_name: rabbitmq
     depends_on:
       certfixer:
@@ -105,16 +105,15 @@ services:
         condition: service_healthy
     entrypoint: >
       /bin/sh -c "
-      /usr/bin/mc config host add s3 http://outbox:9000 minio miniostorage;
+      /usr/bin/mc alias set s3 http://outbox:9000 minio miniostorage;
       /usr/bin/mc mb s3/lega;
-      exit 0;
       "
 
   doa:
     container_name: doa
     build:
       context: ../../sda-doa
-    image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-doa
+    image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-doa
     depends_on:
       postgres:
         condition: service_healthy
@@ -138,7 +137,6 @@ services:
       - CRYPT4GH_PRIVATE_KEY_PATH=test/crypt4gh.sec.pem
       - CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH=test/crypt4gh.pass
       - OUTBOX_TYPE=S3
-      - SSL_ENABLED=false
       - ROOT_CERT_PATH=/certs/ca.crt
       - CERT_PATH=/certs/client.crt
       - CERT_KEY=/certs/client.der
@@ -159,15 +157,17 @@ services:
     volumes:
       - ../../sda-doa/src:/sda-doa/src
       - ../../sda-doa/pom.xml:/sda-doa/pom.xml
-      - ../../sda-doa/settings.xml:/root/.m2/settings.xml
       - ./tests:/tests
       - encryption_files:/test
       - client_certs:/certs
 
     depends_on:
-      - doa
-      - mockauth
-      - init-bucket
+      doa:
+        condition: service_started
+      mockauth:
+        condition: service_started
+      init-bucket:
+        condition: service_completed_successfully
     environment:
       - OUTBOX_TYPE=S3
       - DOA_URL=http://doa:8080