generate dynamic docker-compose.yml for integration

this way, we don't need to worry about the main docker-compose.yml
drifting from the override as much, since the override contains a
complete copy of the worker service. this way, we just clone the
existing service and modify it as needed in code.

also enhances the assertion to ensure we pass a COW volume in as both
the base image and the input

Signed-off-by: Aidan Oldershaw <aoldershaw@pivotal.io>

integration/internal/dctest/cmd.go

@@ -60,9 +60,9 @@ func Init(t *testing.T, composeFile string, overrides ...string) Cmd {
 	}
 }
 
-func InitDynamic(t *testing.T, doc *ypath.Document) Cmd {
+func InitDynamic(t *testing.T, doc *ypath.Document, parentDir string) Cmd {
 	name := filepath.Base(t.Name())
-	fileName := fmt.Sprintf(".docker-compose.%s.yml", name)
+	fileName := filepath.Join(parentDir, fmt.Sprintf(".docker-compose.%s.yml", name))
 
 	err := ioutil.WriteFile(fileName, doc.Bytes(), os.ModePerm)
 	require.NoError(t, err)

integration/internal/ypath/ypath.go

@@ -77,3 +77,33 @@ func (doc *Document) Set(t *testing.T, pathString string, value interface{}) {
 	err = path.MergeFromNode(doc.file, node)
 	require.NoError(t, err)
 }
+
+func (doc *Document) Delete(t *testing.T, pathString string) {
+	segments := strings.Split(pathString, ".")
+	field := segments[len(segments)-1]
+	parent := strings.Join(segments[0:len(segments)-1], ".")
+
+	path, err := yaml.PathString(parent)
+	require.NoError(t, err)
+
+	var newMap map[string]interface{}
+	doc.Read(t, parent, &newMap)
+	delete(newMap, field)
+
+	newNode, err := yaml.NewEncoder(nil).EncodeToNode(newMap)
+	require.NoError(t, err)
+
+	err = path.ReplaceWithNode(doc.file, newNode)
+	require.NoError(t, err)
+}
+
+func (doc *Document) Clone(t *testing.T, srcPath string, dstPath string) {
+	var src interface{}
+	doc.Read(t, srcPath, &src)
+	doc.Set(t, dstPath, src)
+}
+
+func (doc *Document) Move(t *testing.T, srcPath string, dstPath string) {
+	doc.Clone(t, srcPath, dstPath)
+	doc.Delete(t, srcPath)
+}

integration/worker/overrides/streamed_resource_cache.yml

@@ -1,32 +0,0 @@
-version: '3'
-
-services:
-  worker:
-    environment:
-      CONCOURSE_NAME: worker1
-      CONCOURSE_TAG: tag1
-
-  worker2:
-    image: ${TEST_CONCOURSE_DEV_IMAGE:-concourse/concourse:local}
-    command: worker
-    privileged: true
-    depends_on: [web]
-    volumes: ["../hack/keys:/concourse-keys"]
-    stop_signal: SIGUSR2
-    environment:
-      CONCOURSE_RUNTIME: containerd
-
-      CONCOURSE_TSA_PUBLIC_KEY: /concourse-keys/tsa_host_key.pub
-      CONCOURSE_TSA_WORKER_PRIVATE_KEY: /concourse-keys/worker_key
-
-      CONCOURSE_LOG_LEVEL: debug
-      CONCOURSE_TSA_HOST: web:2222
-
-      # avoid using loopbacks
-      CONCOURSE_BAGGAGECLAIM_DRIVER: overlay
-
-      # work with docker-compose's dns
-      CONCOURSE_CONTAINERD_DNS_PROXY_ENABLE: "true"
-
-      CONCOURSE_NAME: worker2
-      CONCOURSE_TAG: tag2

integration/worker/pipelines/streamed_resource_cache.yml

@@ -14,9 +14,26 @@ jobs:
     image: some-image
     config:
       platform: linux
+      inputs: [{name: some-image}]
       run:
         path: sh
         args:
         - -c
-        - echo hello-world
+        - |
+          echo hello-world > /foo
+          echo hello-world > some-image/foo
+    tags: [tag2]
+  - task: task2
+    image: some-image
+    config:
+      platform: linux
+      run:
+        path: sh
+        args:
+        - -c
+        - |
+          set -ex
+          # ensure each task gets a fresh copy of the image and inputs
+          [ ! -f /foo ]
+          [ ! -f some-image/foo ]
     tags: [tag2]

integration/worker/streamed_resource_cache_test.go

@@ -6,13 +6,26 @@ import (
 
 	"github.com/concourse/concourse/integration/internal/dctest"
 	"github.com/concourse/concourse/integration/internal/flytest"
+	"github.com/concourse/concourse/integration/internal/ypath"
 	"github.com/stretchr/testify/require"
 )
 
 func TestStreamed_ResourceCache(t *testing.T) {
 	t.Parallel()
 
-	dc := dctest.Init(t, "../docker-compose.yml", "overrides/streamed_resource_cache.yml")
+	dockerComposeDoc := ypath.Load(t, "../docker-compose.yml")
+
+	// Configure two worker services in docker-compose with distinct names and tags
+	dockerComposeDoc.Move(t, "$.services.worker", "$.services.worker1")
+	dockerComposeDoc.Clone(t, "$.services.worker1", "$.services.worker2")
+
+	dockerComposeDoc.Set(t, "$.services.worker1.environment.CONCOURSE_NAME", "worker1")
+	dockerComposeDoc.Set(t, "$.services.worker1.environment.CONCOURSE_TAG", "tag1")
+
+	dockerComposeDoc.Set(t, "$.services.worker2.environment.CONCOURSE_NAME", "worker2")
+	dockerComposeDoc.Set(t, "$.services.worker2.environment.CONCOURSE_TAG", "tag2")
+
+	dc := dctest.InitDynamic(t, dockerComposeDoc, "..")
 
 	t.Run("deploy with 2 tagged workers", func(t *testing.T) {
 		dc.Run(t, "up", "-d")
@@ -48,4 +61,4 @@ func findWorkersContainingMockResource(t *testing.T, fly flytest.Cmd) []string {
 		}
 	}
 	return workers
-}
+}