ci: rework how we use Docker Compose in CI

* explicitly generate keys; this allows them to be shared between
  upgrades and downgrades
* don't use the dev Dockerfile and don't perform any building
* use an override file to swap out the image for upgrade/downgrade;
  don't use Quickstart
* stop using ephemeral workers, since this caused the workers to
  disappear under load
* set `stop_signal: SIGUSR2` so that workers retire during
  upgrade/downgrade
* pass the latest final concourse image as an input to upgrade/downgrade

Signed-off-by: Alex Suraci <suraci.alex@gmail.com>

Dockerfile

@@ -3,14 +3,6 @@
 
 FROM concourse/dev
 
-# configure key location (they're actually generated at the end)
-RUN mkdir /concourse-keys
-ENV CONCOURSE_TSA_HOST_KEY        /concourse-keys/tsa_host_key
-ENV CONCOURSE_TSA_AUTHORIZED_KEYS /concourse-keys/authorized_worker_keys
-ENV CONCOURSE_SESSION_SIGNING_KEY /concourse-keys/session_signing_key
-ENV CONCOURSE_TSA_PUBLIC_KEY         /concourse-keys/tsa_host_key.pub
-ENV CONCOURSE_TSA_WORKER_PRIVATE_KEY /concourse-keys/worker_key
-
 # download go modules separately so this doesn't re-run on every change
 WORKDIR /src
 COPY go.mod .
@@ -25,6 +17,7 @@ RUN go build -gcflags=all="-N -l" -o /usr/local/concourse/bin/concourse \
 VOLUME /src
 
 # generate keys (with 1024 bits just so they generate faster)
+RUN mkdir -p /concourse-keys
 RUN concourse generate-key -t rsa -b 1024 -f /concourse-keys/session_signing_key
 RUN concourse generate-key -t ssh -b 1024 -f /concourse-keys/tsa_host_key
 RUN concourse generate-key -t ssh -b 1024 -f /concourse-keys/worker_key

ci/dockerfiles/dev/Dockerfile

@@ -68,6 +68,15 @@ ENV CONCOURSE_WORK_DIR /worker-state
 # enable DNS proxy to support Docker's 127.x.x.x DNS server
 ENV CONCOURSE_GARDEN_DNS_PROXY_ENABLE true
 
+# 'web' keys
+ENV CONCOURSE_SESSION_SIGNING_KEY     /concourse-keys/session_signing_key
+ENV CONCOURSE_TSA_AUTHORIZED_KEYS     /concourse-keys/authorized_worker_keys
+ENV CONCOURSE_TSA_HOST_KEY            /concourse-keys/tsa_host_key
+
+# 'worker' keys
+ENV CONCOURSE_TSA_PUBLIC_KEY          /concourse-keys/tsa_host_key.pub
+ENV CONCOURSE_TSA_WORKER_PRIVATE_KEY  /concourse-keys/worker_key
+
 # set $PATH for convenience
 ENV PATH /usr/local/concourse/bin:${PATH}
 

ci/dockerfiles/rc/Dockerfile

@@ -35,5 +35,4 @@ RUN apt update && apt install -y \
 
 COPY --from=assets /usr/local/concourse /usr/local/concourse
 
-
 ENTRYPOINT ["dumb-init", "/usr/local/concourse/bin/concourse"]

ci/overrides/docker-compose.ci.yml

@@ -0,0 +1,16 @@
+version: '3'
+
+services:
+  web:
+    image: concourse/dev:latest
+    volumes: ["./keys:/concourse-keys"]
+
+  worker:
+    image: concourse/dev:latest
+    volumes: ["./keys:/concourse-keys"]
+    environment:
+      # configure a network range that doesn't overlap with the outer worker
+      CONCOURSE_GARDEN_NETWORK_POOL: '10.224.0.0/16'
+
+      # prevent worker from dropping out if the outer worker is overloaded
+      CONCOURSE_EPHEMERAL: 'false'

ci/overrides/docker-compose.latest.yml

@@ -0,0 +1,8 @@
+version: '3'
+
+services:
+  web:
+    image: concourse/concourse:latest
+
+  worker:
+    image: concourse/concourse:latest

ci/pipelines/concourse.yml

@@ -249,6 +249,8 @@ jobs:
     - get: dev-image
       passed: [dev-image]
       params: {format: oci}
+    - get: concourse-image
+      params: {format: oci}
     - get: postgres-image
       params: {format: oci}
     - get: unit-image
@@ -270,6 +272,8 @@ jobs:
     - get: dev-image
       passed: [dev-image]
       params: {format: oci}
+    - get: concourse-image
+      params: {format: oci}
     - get: postgres-image
       params: {format: oci}
     - get: unit-image

ci/tasks/downgrade-test.yml

@@ -12,6 +12,8 @@ inputs:
 - name: concourse
 - name: dev-image
   optional: true
+- name: concourse-image
+  optional: true
 - name: postgres-image
   optional: true
 

ci/tasks/scripts/downgrade-test

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -e -u -x
+set -e -u
 
 export PIPELINE_NAME=test-pipeline
 
@@ -11,25 +11,41 @@ source concourse/ci/tasks/scripts/docker-helpers.sh
 start_docker
 
 [ -d dev-image ] && docker load -i dev-image/image.tar
+[ -d concourse-image ] && docker load -i concourse-image/image.tar
 [ -d postgres-image ] && docker load -i postgres-image/image.tar
 
 cd concourse
 
+# generate keys for the cluster
+./ci/tasks/scripts/generate-keys
+
 # start with rc and set up
-docker-compose up --build -d
+docker-compose \
+  -f docker-compose.yml \
+  -f ci/overrides/docker-compose.ci.yml \
+  up --no-build -d
+
 ./ci/tasks/scripts/create-upgrade-downgrade-pipeline
 
 # perform down migrations
 downgrade_to=$(docker run concourse/concourse:latest migrate --supported-db-version)
-docker-compose stop web
-docker-compose run web migrate --migrate-db-to-version $downgrade_to
+docker-compose exec web concourse migrate --migrate-db-to-version $downgrade_to
 
 # downgrade and verify
-docker-compose -f docker-compose-latest.yml up --remove-orphans -d
+docker-compose \
+  -f docker-compose.yml \
+  -f ci/overrides/docker-compose.ci.yml \
+  -f ci/overrides/docker-compose.latest.yml \
+  up --no-build -d
+
 ./ci/tasks/scripts/verify-upgrade-downgrade-pipeline
 
 # upgrade back and verify
-docker-compose up --build --remove-orphans -d
+docker-compose \
+  -f docker-compose.yml \
+  -f ci/overrides/docker-compose.ci.yml \
+  up --no-build -d
+
 ./ci/tasks/scripts/verify-upgrade-downgrade-pipeline
 
 # run smoke tests

ci/tasks/scripts/generate-keys

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+docker run -v $PWD/keys:/keys concourse/dev generate-key -t rsa -b 1024 -f /keys/session_signing_key
+docker run -v $PWD/keys:/keys concourse/dev generate-key -t ssh -b 1024 -f /keys/tsa_host_key
+docker run -v $PWD/keys:/keys concourse/dev generate-key -t ssh -b 1024 -f /keys/worker_key
+cp keys/worker_key.pub keys/authorized_worker_keys

ci/tasks/scripts/upgrade-test

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -e -u -x
+set -e -u
 
 export PIPELINE_NAME=test-pipeline
 
@@ -11,16 +11,29 @@ source concourse/ci/tasks/scripts/docker-helpers.sh
 start_docker
 
 [ -d dev-image ] && docker load -i dev-image/image.tar
+[ -d concourse-image ] && docker load -i concourse-image/image.tar
 [ -d postgres-image ] && docker load -i postgres-image/image.tar
 
 cd concourse
 
+# generate keys for the cluster
+./ci/tasks/scripts/generate-keys
+
 # start with latest release and set up
-docker-compose -f docker-compose-latest.yml up -d
+docker-compose \
+  -f docker-compose.yml \
+  -f ci/overrides/docker-compose.ci.yml \
+  -f ci/overrides/docker-compose.latest.yml \
+  up --no-build -d
+
 ./ci/tasks/scripts/create-upgrade-downgrade-pipeline
 
 # upgrade and verify
-docker-compose up --build --remove-orphans -d
+docker-compose \
+  -f docker-compose.yml \
+  -f ci/overrides/docker-compose.ci.yml \
+  up --no-build -d
+
 ./ci/tasks/scripts/verify-upgrade-downgrade-pipeline
 
 # run smoke tests

ci/tasks/scripts/with-docker-compose

@@ -10,10 +10,12 @@ start_docker
 [ -d postgres-image ] && docker load -i postgres-image/image.tar
 
 pushd concourse
+  ./ci/tasks/scripts/generate-keys
+
   docker-compose \
     -f docker-compose.yml \
-    -f docker-compose.nested-pool.yml \
-    up -d
+    -f ci/overrides/docker-compose.ci.yml \
+    up --no-build -d
 popd
 
 "$@"

ci/tasks/upgrade-test.yml

@@ -12,6 +12,8 @@ inputs:
 - name: concourse
 - name: dev-image
   optional: true
+- name: concourse-image
+  optional: true
 - name: postgres-image
   optional: true
 

docker-compose-latest.yml

@@ -1,26 +0,0 @@
-version: '3'
-
-services:
-  db:
-    image: postgres
-    environment:
-      POSTGRES_DB: concourse
-      POSTGRES_USER: dev
-      POSTGRES_PASSWORD: dev
-
-  concourse:
-    image: concourse/concourse:latest
-    command: quickstart
-    privileged: true
-    depends_on: [db]
-    ports: ["8080:8080"]
-    environment:
-      CONCOURSE_POSTGRES_HOST: db
-      CONCOURSE_POSTGRES_USER: dev
-      CONCOURSE_POSTGRES_PASSWORD: dev
-      CONCOURSE_POSTGRES_DATABASE: concourse
-      CONCOURSE_EXTERNAL_URL: http://localhost:8080
-      CONCOURSE_ADD_LOCAL_USER: test:test,guest:guest
-      CONCOURSE_MAIN_TEAM_LOCAL_USER: 'test'
-      CONCOURSE_WORKER_EPHEMERAL: 'true'
-      CONCOURSE_LOG_LEVEL: debug

docker-compose.nested-pool.yml

@@ -1,8 +0,0 @@
-version: '3'
-
-services:
-  worker:
-    environment:
-      # so that CI can configure a different network range that doesn't overlap
-      # with the outer Garden
-      CONCOURSE_GARDEN_NETWORK_POOL: '10.224.0.0/16'

docker-compose.yml

@@ -22,6 +22,7 @@ services:
     ports:
     - 8080:8080
     environment:
+      CONCOURSE_LOG_LEVEL: debug
       CONCOURSE_POSTGRES_HOST: db
       CONCOURSE_POSTGRES_USER: dev
       CONCOURSE_POSTGRES_PASSWORD: dev
@@ -29,7 +30,6 @@ services:
       CONCOURSE_EXTERNAL_URL: http://localhost:8080
       CONCOURSE_ADD_LOCAL_USER: test:test,guest:guest
       CONCOURSE_MAIN_TEAM_LOCAL_USER: test
-      CONCOURSE_LOG_LEVEL: debug
 
   worker:
     build:
@@ -42,10 +42,10 @@ services:
     ports:
     - 7777:7777
     - 7788:7788
+    stop_signal: SIGUSR2
     environment:
       CONCOURSE_LOG_LEVEL: debug
       CONCOURSE_TSA_HOST: web:2222
-      CONCOURSE_EPHEMERAL: ${CONCOURSE_EPHEMERAL:-true}
 
       # avoid using loopbacks
       CONCOURSE_BAGGAGECLAIM_DRIVER: overlay