quickwit-metastore-migration/docker-compose.e2e.yml at main · hatemosphere/quickwit-metastore-migration · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
# Full E2E test: index real data → migrate → verify queries match
#
# Usage: ./test/e2e-test.sh
#
# Phase 1: quickwit-file indexes data with file-backed metastore (S3/MinIO)
# Phase 2: migration tool moves metastore to PostgreSQL
# Phase 3: quickwit-postgres serves same data, queries must match
services:
  minio:
    # minio/minio:latest as of 2026-02-10
    image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
    command: server /data --console-address ":9001"
    ports:
      - "9000:9000"
      - "9001:9001"
    environment:
      MINIO_ROOT_USER: minioadmin
      MINIO_ROOT_PASSWORD: minioadmin
    healthcheck:
      test: ["CMD", "mc", "ready", "local"]
      interval: 5s
      timeout: 5s
      retries: 5

  minio-setup:
    # minio/mc:latest as of 2026-02-10
    image: minio/mc@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727
    depends_on:
      minio:
        condition: service_healthy
    entrypoint: ["/bin/sh", "-c"]
    command:
      - |
        mc alias set local http://minio:9000 minioadmin minioadmin
        mc mb --ignore-existing local/test-bucket
        echo "Bucket created."

  postgres:
    # postgres:18 as of 2026-02-10
    image: postgres@sha256:1090bc3a8ccfb0b55f78a494d76f8d603434f7e4553543d6e807bc7bd6bbd17f
    ports:
      - "15432:5432"
    environment:
      POSTGRES_DB: quickwit_metastore
      POSTGRES_USER: quickwit
      POSTGRES_PASSWORD: quickwit
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U quickwit -d quickwit_metastore"]
      interval: 5s
      timeout: 5s
      retries: 5

  # Phase 1: Quickwit with file-backed metastore (for indexing real data)
  quickwit-file:
    # quickwit/quickwit:edge as of 2026-02-10
    image: quickwit/quickwit@sha256:665a4f5344ee3aa3b572746af739775e2026a81cc3f44ce6a38efe762c01e0f2
    depends_on:
      minio-setup:
        condition: service_completed_successfully
    ports:
      - "7280:7280"
      - "7281:7281"
    environment:
      QW_LISTEN_ADDRESS: 0.0.0.0
      AWS_ACCESS_KEY_ID: minioadmin
      AWS_SECRET_ACCESS_KEY: minioadmin
      RUST_LOG: quickwit=info
      QW_DISABLE_TELEMETRY: "1"
    volumes:
      - ./test/qw-file-config.yaml:/quickwit/config/quickwit.yaml:ro
      - qw-data:/quickwit/qwdata
    command: ["run"]
    healthcheck:
      test: ["CMD-SHELL", "curl -sf http://localhost:7280/health/readyz || exit 1"]
      interval: 5s
      timeout: 5s
      retries: 30
      start_period: 10s

  # Phase 2: Migration tool (file-backed → PostgreSQL)
  migration:
    build: .
    depends_on:
      postgres:
        condition: service_healthy
    environment:
      AWS_ACCESS_KEY_ID: minioadmin
      AWS_SECRET_ACCESS_KEY: minioadmin
      RUST_LOG: quickwit_metastore_migration=info,quickwit_metastore=info
    command: >
      --source-config /config/node.yaml
      --target-postgres-url postgresql://quickwit:quickwit@postgres:5432/quickwit_metastore
      --index e2e-test-index
    volumes:
      - ./test/node.yaml:/config/node.yaml:ro

  # Phase 3: Quickwit with PostgreSQL metastore (for verification)
  quickwit-postgres:
    # quickwit/quickwit:edge as of 2026-02-10
    image: quickwit/quickwit@sha256:665a4f5344ee3aa3b572746af739775e2026a81cc3f44ce6a38efe762c01e0f2
    depends_on:
      postgres:
        condition: service_healthy
    ports:
      - "7280:7280"
      - "7281:7281"
    environment:
      QW_LISTEN_ADDRESS: 0.0.0.0
      AWS_ACCESS_KEY_ID: minioadmin
      AWS_SECRET_ACCESS_KEY: minioadmin
      RUST_LOG: quickwit=info
      QW_DISABLE_TELEMETRY: "1"
    volumes:
      - ./test/qw-postgres-config.yaml:/quickwit/config/quickwit.yaml:ro
      - qw-data-pg:/quickwit/qwdata
    command: ["run"]
    healthcheck:
      test: ["CMD-SHELL", "curl -sf http://localhost:7280/health/readyz || exit 1"]
      interval: 5s
      timeout: 5s
      retries: 30
      start_period: 10s

volumes:
  qw-data:
  qw-data-pg: