Merge pull request #4980 from concourse/topgun-race-resiliency

Fix flakey topgun tests

topgun/both/atc_shutdown_test.go

@@ -84,7 +84,10 @@ var _ = Describe("ATC Shutting down", func() {
 				matches := buildRegex.FindSubmatch(buildSession.Out.Contents())
 				buildID = string(matches[1])
 
-				Eventually(buildSession).Should(gbytes.Say("waiting for /tmp/stop-waiting"))
+				//For the initializing block
+				Eventually(buildSession).Should(gbytes.Say("echo 'waiting for /tmp/stop-waiting to exist'"))
+				//For the output from the running step
+				Eventually(buildSession).Should(gbytes.Say("waiting for /tmp/stop-waiting to exist"))
 			})
 
 			Context("when the web node tracking the build shuts down", func() {
@@ -100,14 +103,16 @@ var _ = Describe("ATC Shutting down", func() {
 					time.Sleep(10 * time.Second)
 
 					By("hijacking the build to tell it to finish")
-					hijackSession := Fly.Start(
-						"hijack",
-						"-b", buildID,
-						"-s", "one-off",
-						"touch", "/tmp/stop-waiting",
-					)
-					<-hijackSession.Exited
-					Expect(hijackSession.ExitCode()).To(Equal(0))
+					Eventually(func() int {
+						hijackSession := Fly.Start(
+							"hijack",
+							"-b", buildID,
+							"-s", "one-off", "--",
+							"touch", "/tmp/stop-waiting",
+						)
+						<-hijackSession.Exited
+						return hijackSession.ExitCode()
+					}).Should(Equal(0))
 
 					By("waiting for the build to exit")
 					Eventually(watchSession, 1*time.Minute).Should(gbytes.Say("done"))

topgun/both/hijack_container_gc_test.go

@@ -45,7 +45,10 @@ var _ = Describe("Hijacked containers", func() {
 		By("triggering the build")
 		Fly.Run("unpause-pipeline", "-p", "hijacked-containers-test")
 		buildSession := Fly.Start("trigger-job", "-w", "-j", "hijacked-containers-test/simple-job")
-		Eventually(buildSession).Should(gbytes.Say("waiting for /tmp/stop-waiting"))
+		//For the initializing block
+		Eventually(buildSession).Should(gbytes.Say("echo 'waiting for /tmp/stop-waiting to exist'"))
+		//For the output from the running step
+		Eventually(buildSession).Should(gbytes.Say("waiting for /tmp/stop-waiting to exist"))
 
 		By("hijacking into the build container")
 		hijackSession := Fly.Start(
@@ -57,25 +60,37 @@ var _ = Describe("Hijacked containers", func() {
 		)
 
 		By("finishing the build")
-		<-Fly.Start(
-			"hijack",
-			"-j", "hijacked-containers-test/simple-job",
-			"-b", "1",
-			"-s", "simple-task",
-			"touch", "/tmp/stop-waiting",
-		).Exited
+		Eventually(func() int {
+			hS := Fly.Start(
+				"hijack",
+				"-j", "hijacked-containers-test/simple-job",
+				"-s", "simple-task",
+				"touch", "/tmp/stop-waiting",
+			)
+			<-hS.Exited
+			return hS.ExitCode()
+		}).Should(Equal(0))
+
 		<-buildSession.Exited
 
 		By("triggering a new build")
 		buildSession = Fly.Start("trigger-job", "-w", "-j", "hijacked-containers-test/simple-job")
-		Eventually(buildSession).Should(gbytes.Say("waiting for /tmp/stop-waiting"))
-		<-Fly.Start(
-			"hijack",
-			"-j", "hijacked-containers-test/simple-job",
-			"-b", "2",
-			"-s", "simple-task",
-			"touch", "/tmp/stop-waiting",
-		).Exited
+		//For the initializing block
+		Eventually(buildSession).Should(gbytes.Say("echo 'waiting for /tmp/stop-waiting to exist'"))
+		//For the output from the running step
+		Eventually(buildSession).Should(gbytes.Say("waiting for /tmp/stop-waiting to exist"))
+
+		Eventually(func() int {
+			hS := Fly.Start(
+				"hijack",
+				"-j", "hijacked-containers-test/simple-job",
+				"-b", "2",
+				"-s", "simple-task",
+				"touch", "/tmp/stop-waiting",
+			)
+			<-hS.Exited
+			return hS.ExitCode()
+		}).Should(Equal(0))
 		<-buildSession.Exited
 
 		By("verifying the hijacked container exists via fly and Garden")
@@ -91,7 +106,10 @@ var _ = Describe("Hijacked containers", func() {
 	It("does not delete hijacked one-off build containers from the database, and sets a 5 minute TTL on the container in garden", func() {
 		By("triggering a one-off build")
 		buildSession := Fly.Start("execute", "-c", "tasks/wait.yml")
-		Eventually(buildSession).Should(gbytes.Say("waiting for /tmp/stop-waiting"))
+		//For the initializing block
+		Eventually(buildSession).Should(gbytes.Say("echo 'waiting for /tmp/stop-waiting to exist'"))
+		//For the output from the running step
+		Eventually(buildSession).Should(gbytes.Say("waiting for /tmp/stop-waiting to exist"))
 
 		By("hijacking into the build container")
 		hijackSession := Fly.Start(
@@ -102,11 +120,15 @@ var _ = Describe("Hijacked containers", func() {
 		)
 
 		By("waiting for build to finish")
-		<-Fly.Start(
-			"hijack",
-			"-b", "1",
-			"touch", "/tmp/stop-waiting",
-		).Exited
+		Eventually(func() int {
+			hS := Fly.Start(
+				"hijack",
+				"-b", "1",
+				"touch", "/tmp/stop-waiting",
+			)
+			<-hS.Exited
+			return hS.ExitCode()
+		}).Should(Equal(0))
 		<-buildSession.Exited
 
 		By("verifying the hijacked container exists via fly and Garden")

topgun/both/image_resource_gc_test.go

@@ -46,7 +46,10 @@ var _ = Describe("A build using an image_resource", func() {
 
 			By("triggering a build that waits")
 			watchSession := Fly.Start("trigger-job", "-w", "-j", "test/some-job")
-			Eventually(watchSession).Should(gbytes.Say("waiting for /tmp/stop-waiting"))
+			//For the initializing block
+			Eventually(watchSession).Should(gbytes.Say("echo 'waiting for /tmp/stop-waiting to exist'"))
+			//For the output from the running step
+			Eventually(watchSession).Should(gbytes.Say("waiting for /tmp/stop-waiting to exist"))
 
 			By("getting the resource cache volumes")
 			volumes := FlyTable("volumes")

topgun/both/worker_landing_test.go

@@ -194,7 +194,16 @@ var _ = Describe("Worker landing", func() {
 
 				It("finishes landing once the build is done", func() {
 					By("hijacking the build to tell it to finish")
-					Fly.Run("hijack", "-b", buildID, "-s", "one-off", "--", "touch", "/tmp/stop-waiting")
+					Eventually(func() int {
+						hijackSession := Fly.Start(
+							"hijack",
+							"-b", buildID,
+							"-s", "one-off", "--",
+							"touch", "/tmp/stop-waiting",
+						)
+						<-hijackSession.Exited
+						return hijackSession.ExitCode()
+					}).Should(Equal(0))
 
 					By("waiting for the build to exit")
 					Eventually(buildSession).Should(gbytes.Say("done"))

topgun/both/worker_stalling_test.go

@@ -82,7 +82,10 @@ var _ = Describe("Worker stalling", func() {
 				matches := buildRegex.FindSubmatch(buildSession.Out.Contents())
 				buildID = string(matches[1])
 
-				Eventually(buildSession).Should(gbytes.Say("waiting for /tmp/stop-waiting"))
+				//For the initializing block
+				Eventually(buildSession).Should(gbytes.Say("echo 'waiting for /tmp/stop-waiting to exist'"))
+				//For the output from the running step
+				Eventually(buildSession).Should(gbytes.Say("waiting for /tmp/stop-waiting to exist"))
 
 				By("stopping the worker without draining")
 				Bosh("ssh", "worker/0", "-c", "sudo /var/vcap/bosh/bin/monit stop worker")
@@ -118,7 +121,7 @@ var _ = Describe("Worker stalling", func() {
 					Expect(err).ToNot(HaveOccurred())
 
 					// wait for new output
-					Eventually(buildSession).Should(gbytes.Say("waiting for /tmp/stop-waiting"))
+					Eventually(buildSession).Should(gbytes.Say("waiting for /tmp/stop-waiting to exist"))
 
 					By("hijacking the build to tell it to finish")
 					Eventually(func() int {

topgun/core/containers_scoped_to_team_test.go

@@ -63,15 +63,18 @@ var _ = Describe("Container scope", func() {
 			Fly.Run("login", "-n", "main", "-u", AtcUsername, "-p", AtcPassword)
 
 			By("stopping the build")
-			hijackSession := Fly.Start(
-				"hijack",
-				"-b", "1",
-				"-s", "simple-task",
-				"touch", "/tmp/stop-waiting",
-			)
+			Eventually(func() int {
+				hijackSession := Fly.Start(
+					"hijack",
+					"-b", "1",
+					"-s", "simple-task",
+					"touch", "/tmp/stop-waiting",
+				)
 
-			<-hijackSession.Exited
-			Expect(hijackSession.ExitCode()).To(Equal(0))
+				<-hijackSession.Exited
+				return hijackSession.ExitCode()
+
+			}).Should(Equal(0))
 
 			Eventually(buildSession).Should(gbytes.Say("done"))
 			<-buildSession.Exited

topgun/core/syslog_test.go

@@ -30,11 +30,10 @@ var _ = Describe("An ATC with syslog draining set", func() {
 		<-buildSession.Exited
 		Expect(buildSession.ExitCode()).To(Equal(0))
 
-		Bosh("scp", "web/0:/var/vcap/store/syslog_storer/syslog.log", "/tmp/syslog.log")
-		found, err := checkContent("/tmp/syslog.log", "shhhh")
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(found).To(BeTrue())
+		Eventually(func() (bool, error) {
+			Bosh("scp", "web/0:/var/vcap/store/syslog_storer/syslog.log", "/tmp/syslog.log")
+			return checkContent("/tmp/syslog.log", "shhhh")
+		}).Should(BeTrue())
 	})
 })