opensourcemirror
/
concourse
mirror of https://github.com/concourse/concourse.git
1
0
Fork 0
Code Issues Releases Wiki Activity

feat: redacting secrets in build logs. 

With this change, for all secrets fetched from credential manager, if they appear
in build logs, then they will be automatically redacted as "[**redacted**]]".

As this is currently an exprimental feature, a "concourse web" command option
"--enable-redact-secrets" has been added to turn on this feature.

Close #4311

Signed-off-by: Chao Li <chaol@vmware.com>
This commit is contained in:
Chao Li 2019-08-22 10:38:00 -07:00
parent ba9bad2c15
commit ebc23a7fa5
24 changed files with 852 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
atc/atccmd/command.go
View File

@ -195,6 +195,8 @@ type RunCommand struct {
AuthFlags skycmd.AuthFlags
MainTeamFlags skycmd.AuthTeamFlags `group:"Authentication (Main Team)" namespace:"main-team"`
} `group:"Authentication"`
EnableRedactSecrets bool `long:"enable-redact-secrets" description:"Enable redacting secrets in build logs."`
}
var HelpError = errors.New("must specify one of `--current-db-version`, `--supported-db-version`, or `--migrate-db-to-version`")
@ -1301,7 +1303,6 @@ func (cmd *RunCommand) constructEngine(
resourceFetcher,
resourceCacheFactory,
resourceConfigFactory,
secretManager,
defaultLimits,
strategy,
resourceFactory,
@ -1312,6 +1313,8 @@ func (cmd *RunCommand) constructEngine(
stepFactory,
builder.NewDelegateFactory(),
cmd.ExternalURL.String(),
secretManager,
cmd.EnableRedactSecrets,
)
return engine.NewEngine(stepBuilder)

128
atc/engine/builder/builder.go
View File

@ -6,8 +6,10 @@ import (
"strings"
"github.com/concourse/concourse/atc"
"github.com/concourse/concourse/atc/creds"
"github.com/concourse/concourse/atc/db"
"github.com/concourse/concourse/atc/exec"
"github.com/concourse/concourse/vars"
)
const supportedSchema = "exec.v2"
@ -26,22 +28,26 @@ type StepFactory interface {
//go:generate counterfeiter . DelegateFactory
type DelegateFactory interface {
GetDelegate(db.Build, atc.PlanID) exec.GetDelegate
PutDelegate(db.Build, atc.PlanID) exec.PutDelegate
TaskDelegate(db.Build, atc.PlanID) exec.TaskDelegate
GetDelegate(db.Build, atc.PlanID, vars.CredVarsTracker) exec.GetDelegate
PutDelegate(db.Build, atc.PlanID, vars.CredVarsTracker) exec.PutDelegate
TaskDelegate(db.Build, atc.PlanID, vars.CredVarsTracker) exec.TaskDelegate
CheckDelegate(db.Check, atc.PlanID) exec.CheckDelegate
BuildStepDelegate(db.Build, atc.PlanID) exec.BuildStepDelegate
BuildStepDelegate(db.Build, atc.PlanID, vars.CredVarsTracker) exec.BuildStepDelegate
}
func NewStepBuilder(
stepFactory StepFactory,
delegateFactory DelegateFactory,
externalURL string,
secrets creds.Secrets,
redactSecrets bool,
) *stepBuilder {
return &stepBuilder{
stepFactory: stepFactory,
delegateFactory: delegateFactory,
externalURL: externalURL,
secrets: secrets,
redactSecrets: redactSecrets,
}
}
@ -49,10 +55,11 @@ type stepBuilder struct {
stepFactory StepFactory
delegateFactory DelegateFactory
externalURL string
secrets creds.Secrets
redactSecrets bool
}
func (builder *stepBuilder) BuildStep(build db.Build) (exec.Step, error) {
if build == nil {
return exec.IdentityStep{}, errors.New("Must provide a build")
}
@ -61,7 +68,8 @@ func (builder *stepBuilder) BuildStep(build db.Build) (exec.Step, error) {
return exec.IdentityStep{}, errors.New("Schema not supported")
}
return builder.buildStep(build, build.PrivatePlan()), nil
credVarsTracker := vars.NewCredVarsTracker(creds.NewVariables(builder.secrets, build.TeamName(), build.PipelineName()), builder.redactSecrets)
return builder.buildStep(build, build.PrivatePlan(), credVarsTracker), nil
}
func (builder *stepBuilder) CheckStep(check db.Check) (exec.Step, error) {
@ -77,182 +85,182 @@ func (builder *stepBuilder) CheckStep(check db.Check) (exec.Step, error) {
return builder.buildCheckStep(check, check.Plan()), nil
}
func (builder *stepBuilder) buildStep(build db.Build, plan atc.Plan) exec.Step {
func (builder *stepBuilder) buildStep(build db.Build, plan atc.Plan, credVarsTracker vars.CredVarsTracker) exec.Step {
if plan.Aggregate != nil {
return builder.buildAggregateStep(build, plan)
return builder.buildAggregateStep(build, plan, credVarsTracker)
}
if plan.InParallel != nil {
return builder.buildParallelStep(build, plan)
return builder.buildParallelStep(build, plan, credVarsTracker)
}
if plan.Do != nil {
return builder.buildDoStep(build, plan)
return builder.buildDoStep(build, plan, credVarsTracker)
}
if plan.Timeout != nil {
return builder.buildTimeoutStep(build, plan)
return builder.buildTimeoutStep(build, plan, credVarsTracker)
}
if plan.Try != nil {
return builder.buildTryStep(build, plan)
return builder.buildTryStep(build, plan, credVarsTracker)
}
if plan.OnAbort != nil {
return builder.buildOnAbortStep(build, plan)
return builder.buildOnAbortStep(build, plan, credVarsTracker)
}
if plan.OnError != nil {
return builder.buildOnErrorStep(build, plan)
return builder.buildOnErrorStep(build, plan, credVarsTracker)
}
if plan.OnSuccess != nil {
return builder.buildOnSuccessStep(build, plan)
return builder.buildOnSuccessStep(build, plan, credVarsTracker)
}
if plan.OnFailure != nil {
return builder.buildOnFailureStep(build, plan)
return builder.buildOnFailureStep(build, plan, credVarsTracker)
}
if plan.Ensure != nil {
return builder.buildEnsureStep(build, plan)
return builder.buildEnsureStep(build, plan, credVarsTracker)
}
if plan.Task != nil {
return builder.buildTaskStep(build, plan)
return builder.buildTaskStep(build, plan, credVarsTracker)
}
if plan.Get != nil {
return builder.buildGetStep(build, plan)
return builder.buildGetStep(build, plan, credVarsTracker)
}
if plan.Put != nil {
return builder.buildPutStep(build, plan)
return builder.buildPutStep(build, plan, credVarsTracker)
}
if plan.Retry != nil {
return builder.buildRetryStep(build, plan)
return builder.buildRetryStep(build, plan, credVarsTracker)
}
if plan.ArtifactInput != nil {
return builder.buildArtifactInputStep(build, plan)
return builder.buildArtifactInputStep(build, plan, credVarsTracker)
}
if plan.ArtifactOutput != nil {
return builder.buildArtifactOutputStep(build, plan)
return builder.buildArtifactOutputStep(build, plan, credVarsTracker)
}
return exec.IdentityStep{}
}
func (builder *stepBuilder) buildAggregateStep(build db.Build, plan atc.Plan) exec.Step {
func (builder *stepBuilder) buildAggregateStep(build db.Build, plan atc.Plan, credVarsTracker vars.CredVarsTracker) exec.Step {
agg := exec.AggregateStep{}
for _, innerPlan := range *plan.Aggregate {
innerPlan.Attempts = plan.Attempts
step := builder.buildStep(build, innerPlan)
step := builder.buildStep(build, innerPlan, credVarsTracker)
agg = append(agg, step)
}
return agg
}
func (builder *stepBuilder) buildParallelStep(build db.Build, plan atc.Plan) exec.Step {
func (builder *stepBuilder) buildParallelStep(build db.Build, plan atc.Plan, credVarsTracker vars.CredVarsTracker) exec.Step {
var steps []exec.Step
for _, innerPlan := range plan.InParallel.Steps {
innerPlan.Attempts = plan.Attempts
step := builder.buildStep(build, innerPlan)
step := builder.buildStep(build, innerPlan, credVarsTracker)
steps = append(steps, step)
}
return exec.InParallel(steps, plan.InParallel.Limit, plan.InParallel.FailFast)
}
func (builder *stepBuilder) buildDoStep(build db.Build, plan atc.Plan) exec.Step {
func (builder *stepBuilder) buildDoStep(build db.Build, plan atc.Plan, credVarsTracker vars.CredVarsTracker) exec.Step {
var step exec.Step = exec.IdentityStep{}
for i := len(*plan.Do) - 1; i >= 0; i-- {
innerPlan := (*plan.Do)[i]
innerPlan.Attempts = plan.Attempts
previous := builder.buildStep(build, innerPlan)
previous := builder.buildStep(build, innerPlan, credVarsTracker)
step = exec.OnSuccess(previous, step)
}
return step
}
func (builder *stepBuilder) buildTimeoutStep(build db.Build, plan atc.Plan) exec.Step {
func (builder *stepBuilder) buildTimeoutStep(build db.Build, plan atc.Plan, credVarsTracker vars.CredVarsTracker) exec.Step {
innerPlan := plan.Timeout.Step
innerPlan.Attempts = plan.Attempts
step := builder.buildStep(build, innerPlan)
step := builder.buildStep(build, innerPlan, credVarsTracker)
return exec.Timeout(step, plan.Timeout.Duration)
}
func (builder *stepBuilder) buildTryStep(build db.Build, plan atc.Plan) exec.Step {
func (builder *stepBuilder) buildTryStep(build db.Build, plan atc.Plan, credVarsTracker vars.CredVarsTracker) exec.Step {
innerPlan := plan.Try.Step
innerPlan.Attempts = plan.Attempts
step := builder.buildStep(build, innerPlan)
step := builder.buildStep(build, innerPlan, credVarsTracker)
return exec.Try(step)
}
func (builder *stepBuilder) buildOnAbortStep(build db.Build, plan atc.Plan) exec.Step {
func (builder *stepBuilder) buildOnAbortStep(build db.Build, plan atc.Plan, credVarsTracker vars.CredVarsTracker) exec.Step {
plan.OnAbort.Step.Attempts = plan.Attempts
step := builder.buildStep(build, plan.OnAbort.Step)
step := builder.buildStep(build, plan.OnAbort.Step, credVarsTracker)
plan.OnAbort.Next.Attempts = plan.Attempts
next := builder.buildStep(build, plan.OnAbort.Next)
next := builder.buildStep(build, plan.OnAbort.Next, credVarsTracker)
return exec.OnAbort(step, next)
}
func (builder *stepBuilder) buildOnErrorStep(build db.Build, plan atc.Plan) exec.Step {
func (builder *stepBuilder) buildOnErrorStep(build db.Build, plan atc.Plan, credVarsTracker vars.CredVarsTracker) exec.Step {
plan.OnError.Step.Attempts = plan.Attempts
step := builder.buildStep(build, plan.OnError.Step)
step := builder.buildStep(build, plan.OnError.Step, credVarsTracker)
plan.OnError.Next.Attempts = plan.Attempts
next := builder.buildStep(build, plan.OnError.Next)
next := builder.buildStep(build, plan.OnError.Next, credVarsTracker)
return exec.OnError(step, next)
}
func (builder *stepBuilder) buildOnSuccessStep(build db.Build, plan atc.Plan) exec.Step {
func (builder *stepBuilder) buildOnSuccessStep(build db.Build, plan atc.Plan, credVarsTracker vars.CredVarsTracker) exec.Step {
plan.OnSuccess.Step.Attempts = plan.Attempts
step := builder.buildStep(build, plan.OnSuccess.Step)
step := builder.buildStep(build, plan.OnSuccess.Step, credVarsTracker)
plan.OnSuccess.Next.Attempts = plan.Attempts
next := builder.buildStep(build, plan.OnSuccess.Next)
next := builder.buildStep(build, plan.OnSuccess.Next, credVarsTracker)
return exec.OnSuccess(step, next)
}
func (builder *stepBuilder) buildOnFailureStep(build db.Build, plan atc.Plan) exec.Step {
func (builder *stepBuilder) buildOnFailureStep(build db.Build, plan atc.Plan, credVarsTracker vars.CredVarsTracker) exec.Step {
plan.OnFailure.Step.Attempts = plan.Attempts
step := builder.buildStep(build, plan.OnFailure.Step)
step := builder.buildStep(build, plan.OnFailure.Step, credVarsTracker)
plan.OnFailure.Next.Attempts = plan.Attempts
next := builder.buildStep(build, plan.OnFailure.Next)
next := builder.buildStep(build, plan.OnFailure.Next, credVarsTracker)
return exec.OnFailure(step, next)
}
func (builder *stepBuilder) buildEnsureStep(build db.Build, plan atc.Plan) exec.Step {
func (builder *stepBuilder) buildEnsureStep(build db.Build, plan atc.Plan, credVarsTracker vars.CredVarsTracker) exec.Step {
plan.Ensure.Step.Attempts = plan.Attempts
step := builder.buildStep(build, plan.Ensure.Step)
step := builder.buildStep(build, plan.Ensure.Step, credVarsTracker)
plan.Ensure.Next.Attempts = plan.Attempts
next := builder.buildStep(build, plan.Ensure.Next)
next := builder.buildStep(build, plan.Ensure.Next, credVarsTracker)
return exec.Ensure(step, next)
}
func (builder *stepBuilder) buildRetryStep(build db.Build, plan atc.Plan) exec.Step {
func (builder *stepBuilder) buildRetryStep(build db.Build, plan atc.Plan, credVarsTracker vars.CredVarsTracker) exec.Step {
steps := []exec.Step{}
for index, innerPlan := range *plan.Retry {
innerPlan.Attempts = append(plan.Attempts, index+1)
step := builder.buildStep(build, innerPlan)
step := builder.buildStep(build, innerPlan, credVarsTracker)
steps = append(steps, step)
}
return exec.Retry(steps...)
}
func (builder *stepBuilder) buildGetStep(build db.Build, plan atc.Plan) exec.Step {
func (builder *stepBuilder) buildGetStep(build db.Build, plan atc.Plan, credVarsTracker vars.CredVarsTracker) exec.Step {
containerMetadata := builder.containerMetadata(
build,
@ -270,11 +278,11 @@ func (builder *stepBuilder) buildGetStep(build db.Build, plan atc.Plan) exec.Ste
plan,
stepMetadata,
containerMetadata,
builder.delegateFactory.GetDelegate(build, plan.ID),
builder.delegateFactory.GetDelegate(build, plan.ID, credVarsTracker),
)
}
func (builder *stepBuilder) buildPutStep(build db.Build, plan atc.Plan) exec.Step {
func (builder *stepBuilder) buildPutStep(build db.Build, plan atc.Plan, credVarsTracker vars.CredVarsTracker) exec.Step {
containerMetadata := builder.containerMetadata(
build,
@ -292,7 +300,7 @@ func (builder *stepBuilder) buildPutStep(build db.Build, plan atc.Plan) exec.Ste
plan,
stepMetadata,
containerMetadata,
builder.delegateFactory.PutDelegate(build, plan.ID),
builder.delegateFactory.PutDelegate(build, plan.ID, credVarsTracker),
)
}
@ -318,7 +326,7 @@ func (builder *stepBuilder) buildCheckStep(check db.Check, plan atc.Plan) exec.S
)
}
func (builder *stepBuilder) buildTaskStep(build db.Build, plan atc.Plan) exec.Step {
func (builder *stepBuilder) buildTaskStep(build db.Build, plan atc.Plan, credVarsTracker vars.CredVarsTracker) exec.Step {
containerMetadata := builder.containerMetadata(
build,
@ -336,25 +344,25 @@ func (builder *stepBuilder) buildTaskStep(build db.Build, plan atc.Plan) exec.St
plan,
stepMetadata,
containerMetadata,
builder.delegateFactory.TaskDelegate(build, plan.ID),
builder.delegateFactory.TaskDelegate(build, plan.ID, credVarsTracker),
)
}
func (builder *stepBuilder) buildArtifactInputStep(build db.Build, plan atc.Plan) exec.Step {
func (builder *stepBuilder) buildArtifactInputStep(build db.Build, plan atc.Plan, credVarsTracker vars.CredVarsTracker) exec.Step {
return builder.stepFactory.ArtifactInputStep(
plan,
build,
builder.delegateFactory.BuildStepDelegate(build, plan.ID),
builder.delegateFactory.BuildStepDelegate(build, plan.ID, credVarsTracker),
)
}
func (builder *stepBuilder) buildArtifactOutputStep(build db.Build, plan atc.Plan) exec.Step {
func (builder *stepBuilder) buildArtifactOutputStep(build db.Build, plan atc.Plan, credVarsTracker vars.CredVarsTracker) exec.Step {
return builder.stepFactory.ArtifactOutputStep(
plan,
build,
builder.delegateFactory.BuildStepDelegate(build, plan.ID),
builder.delegateFactory.BuildStepDelegate(build, plan.ID, credVarsTracker),
)
}

15
atc/engine/builder/builder_test.go
View File

@ -1,15 +1,16 @@
package builder_test
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/concourse/concourse/atc"
"github.com/concourse/concourse/atc/creds/credsfakes"
"github.com/concourse/concourse/atc/db"
"github.com/concourse/concourse/atc/db/dbfakes"
"github.com/concourse/concourse/atc/engine/builder"
"github.com/concourse/concourse/atc/engine/builder/builderfakes"
"github.com/concourse/concourse/atc/exec"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
type StepBuilder interface {
@ -26,6 +27,7 @@ var _ = Describe("Builder", func() {
fakeStepFactory *builderfakes.FakeStepFactory
fakeDelegateFactory *builderfakes.FakeDelegateFactory
fakeSecretManager *credsfakes.FakeSecrets
planFactory atc.PlanFactory
stepBuilder StepBuilder
@ -34,11 +36,14 @@ var _ = Describe("Builder", func() {
BeforeEach(func() {
fakeStepFactory = new(builderfakes.FakeStepFactory)
fakeDelegateFactory = new(builderfakes.FakeDelegateFactory)
fakeSecretManager = new(credsfakes.FakeSecrets)
stepBuilder = builder.NewStepBuilder(
fakeStepFactory,
fakeDelegateFactory,
"http://example.com",
fakeSecretManager,
false,
)
planFactory = atc.NewPlanFactory(123)
@ -755,6 +760,7 @@ var _ = Describe("Builder", func() {
fakeStepFactory *builderfakes.FakeStepFactory
fakeDelegateFactory *builderfakes.FakeDelegateFactory
fakeSecretManager *credsfakes.FakeSecrets
planFactory atc.PlanFactory
stepBuilder StepBuilder
@ -763,11 +769,14 @@ var _ = Describe("Builder", func() {
BeforeEach(func() {
fakeStepFactory = new(builderfakes.FakeStepFactory)
fakeDelegateFactory = new(builderfakes.FakeDelegateFactory)
fakeSecretManager = new(credsfakes.FakeSecrets)
stepBuilder = builder.NewStepBuilder(
fakeStepFactory,
fakeDelegateFactory,
"http://example.com",
fakeSecretManager,
false,
)
planFactory = atc.NewPlanFactory(123)

73
atc/engine/builder/builderfakes/fake_delegate_factory.go
View File

@ -8,14 +8,16 @@ import (
"github.com/concourse/concourse/atc/db"
"github.com/concourse/concourse/atc/engine/builder"
"github.com/concourse/concourse/atc/exec"
"github.com/concourse/concourse/vars"
)
type FakeDelegateFactory struct {
BuildStepDelegateStub func(db.Build, atc.PlanID) exec.BuildStepDelegate
BuildStepDelegateStub func(db.Build, atc.PlanID, vars.CredVarsTracker) exec.BuildStepDelegate
buildStepDelegateMutex sync.RWMutex
buildStepDelegateArgsForCall []struct {
arg1 db.Build
arg2 atc.PlanID
arg3 vars.CredVarsTracker
}
buildStepDelegateReturns struct {
result1 exec.BuildStepDelegate
@ -35,11 +37,12 @@ type FakeDelegateFactory struct {
checkDelegateReturnsOnCall map[int]struct {
result1 exec.CheckDelegate
}
GetDelegateStub func(db.Build, atc.PlanID) exec.GetDelegate
GetDelegateStub func(db.Build, atc.PlanID, vars.CredVarsTracker) exec.GetDelegate
getDelegateMutex sync.RWMutex
getDelegateArgsForCall []struct {
arg1 db.Build
arg2 atc.PlanID
arg3 vars.CredVarsTracker
}
getDelegateReturns struct {
result1 exec.GetDelegate
@ -47,11 +50,12 @@ type FakeDelegateFactory struct {
getDelegateReturnsOnCall map[int]struct {
result1 exec.GetDelegate
}
PutDelegateStub func(db.Build, atc.PlanID) exec.PutDelegate
PutDelegateStub func(db.Build, atc.PlanID, vars.CredVarsTracker) exec.PutDelegate
putDelegateMutex sync.RWMutex
putDelegateArgsForCall []struct {
arg1 db.Build
arg2 atc.PlanID
arg3 vars.CredVarsTracker
}
putDelegateReturns struct {
result1 exec.PutDelegate
@ -59,11 +63,12 @@ type FakeDelegateFactory struct {
putDelegateReturnsOnCall map[int]struct {
result1 exec.PutDelegate
}
TaskDelegateStub func(db.Build, atc.PlanID) exec.TaskDelegate
TaskDelegateStub func(db.Build, atc.PlanID, vars.CredVarsTracker) exec.TaskDelegate
taskDelegateMutex sync.RWMutex
taskDelegateArgsForCall []struct {
arg1 db.Build
arg2 atc.PlanID
arg3 vars.CredVarsTracker
}
taskDelegateReturns struct {
result1 exec.TaskDelegate
@ -75,17 +80,18 @@ type FakeDelegateFactory struct {
invocationsMutex sync.RWMutex
}
func (fake *FakeDelegateFactory) BuildStepDelegate(arg1 db.Build, arg2 atc.PlanID) exec.BuildStepDelegate {
func (fake *FakeDelegateFactory) BuildStepDelegate(arg1 db.Build, arg2 atc.PlanID, arg3 vars.CredVarsTracker) exec.BuildStepDelegate {
fake.buildStepDelegateMutex.Lock()
ret, specificReturn := fake.buildStepDelegateReturnsOnCall[len(fake.buildStepDelegateArgsForCall)]
fake.buildStepDelegateArgsForCall = append(fake.buildStepDelegateArgsForCall, struct {
arg1 db.Build
arg2 atc.PlanID
}{arg1, arg2})
fake.recordInvocation("BuildStepDelegate", []interface{}{arg1, arg2})
arg3 vars.CredVarsTracker
}{arg1, arg2, arg3})
fake.recordInvocation("BuildStepDelegate", []interface{}{arg1, arg2, arg3})
fake.buildStepDelegateMutex.Unlock()
if fake.BuildStepDelegateStub != nil {
return fake.BuildStepDelegateStub(arg1, arg2)
return fake.BuildStepDelegateStub(arg1, arg2, arg3)
}
if specificReturn {
return ret.result1
@ -100,17 +106,17 @@ func (fake *FakeDelegateFactory) BuildStepDelegateCallCount() int {
return len(fake.buildStepDelegateArgsForCall)
}
func (fake *FakeDelegateFactory) BuildStepDelegateCalls(stub func(db.Build, atc.PlanID) exec.BuildStepDelegate) {
func (fake *FakeDelegateFactory) BuildStepDelegateCalls(stub func(db.Build, atc.PlanID, vars.CredVarsTracker) exec.BuildStepDelegate) {
fake.buildStepDelegateMutex.Lock()
defer fake.buildStepDelegateMutex.Unlock()
fake.BuildStepDelegateStub = stub
}
func (fake *FakeDelegateFactory) BuildStepDelegateArgsForCall(i int) (db.Build, atc.PlanID) {
func (fake *FakeDelegateFactory) BuildStepDelegateArgsForCall(i int) (db.Build, atc.PlanID, vars.CredVarsTracker) {
fake.buildStepDelegateMutex.RLock()
defer fake.buildStepDelegateMutex.RUnlock()
argsForCall := fake.buildStepDelegateArgsForCall[i]
return argsForCall.arg1, argsForCall.arg2
return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3
}
func (fake *FakeDelegateFactory) BuildStepDelegateReturns(result1 exec.BuildStepDelegate) {
@ -197,17 +203,18 @@ func (fake *FakeDelegateFactory) CheckDelegateReturnsOnCall(i int, result1 exec.
}{result1}
}
func (fake *FakeDelegateFactory) GetDelegate(arg1 db.Build, arg2 atc.PlanID) exec.GetDelegate {
func (fake *FakeDelegateFactory) GetDelegate(arg1 db.Build, arg2 atc.PlanID, arg3 vars.CredVarsTracker) exec.GetDelegate {
fake.getDelegateMutex.Lock()
ret, specificReturn := fake.getDelegateReturnsOnCall[len(fake.getDelegateArgsForCall)]
fake.getDelegateArgsForCall = append(fake.getDelegateArgsForCall, struct {
arg1 db.Build
arg2 atc.PlanID
}{arg1, arg2})
fake.recordInvocation("GetDelegate", []interface{}{arg1, arg2})
arg3 vars.CredVarsTracker
}{arg1, arg2, arg3})
fake.recordInvocation("GetDelegate", []interface{}{arg1, arg2, arg3})
fake.getDelegateMutex.Unlock()
if fake.GetDelegateStub != nil {
return fake.GetDelegateStub(arg1, arg2)
return fake.GetDelegateStub(arg1, arg2, arg3)
}
if specificReturn {
return ret.result1
@ -222,17 +229,17 @@ func (fake *FakeDelegateFactory) GetDelegateCallCount() int {
return len(fake.getDelegateArgsForCall)
}
func (fake *FakeDelegateFactory) GetDelegateCalls(stub func(db.Build, atc.PlanID) exec.GetDelegate) {
func (fake *FakeDelegateFactory) GetDelegateCalls(stub func(db.Build, atc.PlanID, vars.CredVarsTracker) exec.GetDelegate) {
fake.getDelegateMutex.Lock()
defer fake.getDelegateMutex.Unlock()
fake.GetDelegateStub = stub
}
func (fake *FakeDelegateFactory) GetDelegateArgsForCall(i int) (db.Build, atc.PlanID) {
func (fake *FakeDelegateFactory) GetDelegateArgsForCall(i int) (db.Build, atc.PlanID, vars.CredVarsTracker) {
fake.getDelegateMutex.RLock()
defer fake.getDelegateMutex.RUnlock()
argsForCall := fake.getDelegateArgsForCall[i]
return argsForCall.arg1, argsForCall.arg2
return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3
}
func (fake *FakeDelegateFactory) GetDelegateReturns(result1 exec.GetDelegate) {
@ -258,17 +265,18 @@ func (fake *FakeDelegateFactory) GetDelegateReturnsOnCall(i int, result1 exec.Ge
}{result1}
}
func (fake *FakeDelegateFactory) PutDelegate(arg1 db.Build, arg2 atc.PlanID) exec.PutDelegate {
func (fake *FakeDelegateFactory) PutDelegate(arg1 db.Build, arg2 atc.PlanID, arg3 vars.CredVarsTracker) exec.PutDelegate {
fake.putDelegateMutex.Lock()
ret, specificReturn := fake.putDelegateReturnsOnCall[len(fake.putDelegateArgsForCall)]
fake.putDelegateArgsForCall = append(fake.putDelegateArgsForCall, struct {
arg1 db.Build
arg2 atc.PlanID
}{arg1, arg2})
fake.recordInvocation("PutDelegate", []interface{}{arg1, arg2})
arg3 vars.CredVarsTracker
}{arg1, arg2, arg3})
fake.recordInvocation("PutDelegate", []interface{}{arg1, arg2, arg3})
fake.putDelegateMutex.Unlock()
if fake.PutDelegateStub != nil {
return fake.PutDelegateStub(arg1, arg2)
return fake.PutDelegateStub(arg1, arg2, arg3)
}
if specificReturn {
return ret.result1
@ -283,17 +291,17 @@ func (fake *FakeDelegateFactory) PutDelegateCallCount() int {
return len(fake.putDelegateArgsForCall)
}
func (fake *FakeDelegateFactory) PutDelegateCalls(stub func(db.Build, atc.PlanID) exec.PutDelegate) {
func (fake *FakeDelegateFactory) PutDelegateCalls(stub func(db.Build, atc.PlanID, vars.CredVarsTracker) exec.PutDelegate) {
fake.putDelegateMutex.Lock()
defer fake.putDelegateMutex.Unlock()
fake.PutDelegateStub = stub
}
func (fake *FakeDelegateFactory) PutDelegateArgsForCall(i int) (db.Build, atc.PlanID) {
func (fake *FakeDelegateFactory) PutDelegateArgsForCall(i int) (db.Build, atc.PlanID, vars.CredVarsTracker) {
fake.putDelegateMutex.RLock()
defer fake.putDelegateMutex.RUnlock()
argsForCall := fake.putDelegateArgsForCall[i]
return argsForCall.arg1, argsForCall.arg2
return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3
}
func (fake *FakeDelegateFactory) PutDelegateReturns(result1 exec.PutDelegate) {
@ -319,17 +327,18 @@ func (fake *FakeDelegateFactory) PutDelegateReturnsOnCall(i int, result1 exec.Pu
}{result1}
}
func (fake *FakeDelegateFactory) TaskDelegate(arg1 db.Build, arg2 atc.PlanID) exec.TaskDelegate {
func (fake *FakeDelegateFactory) TaskDelegate(arg1 db.Build, arg2 atc.PlanID, arg3 vars.CredVarsTracker) exec.TaskDelegate {
fake.taskDelegateMutex.Lock()
ret, specificReturn := fake.taskDelegateReturnsOnCall[len(fake.taskDelegateArgsForCall)]
fake.taskDelegateArgsForCall = append(fake.taskDelegateArgsForCall, struct {
arg1 db.Build
arg2 atc.PlanID
}{arg1, arg2})
fake.recordInvocation("TaskDelegate", []interface{}{arg1, arg2})
arg3 vars.CredVarsTracker
}{arg1, arg2, arg3})
fake.recordInvocation("TaskDelegate", []interface{}{arg1, arg2, arg3})
fake.taskDelegateMutex.Unlock()
if fake.TaskDelegateStub != nil {
return fake.TaskDelegateStub(arg1, arg2)
return fake.TaskDelegateStub(arg1, arg2, arg3)
}
if specificReturn {
return ret.result1
@ -344,17 +353,17 @@ func (fake *FakeDelegateFactory) TaskDelegateCallCount() int {
return len(fake.taskDelegateArgsForCall)
}
func (fake *FakeDelegateFactory) TaskDelegateCalls(stub func(db.Build, atc.PlanID) exec.TaskDelegate) {
func (fake *FakeDelegateFactory) TaskDelegateCalls(stub func(db.Build, atc.PlanID, vars.CredVarsTracker) exec.TaskDelegate) {
fake.taskDelegateMutex.Lock()
defer fake.taskDelegateMutex.Unlock()
fake.TaskDelegateStub = stub
}
func (fake *FakeDelegateFactory) TaskDelegateArgsForCall(i int) (db.Build, atc.PlanID) {
func (fake *FakeDelegateFactory) TaskDelegateArgsForCall(i int) (db.Build, atc.PlanID, vars.CredVarsTracker) {
fake.taskDelegateMutex.RLock()
defer fake.taskDelegateMutex.RUnlock()
argsForCall := fake.taskDelegateArgsForCall[i]
return argsForCall.arg1, argsForCall.arg2
return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3
}
func (fake *FakeDelegateFactory) TaskDelegateReturns(result1 exec.TaskDelegate) {

79
atc/engine/builder/delegate_factory.go
View File

@ -3,15 +3,18 @@ package builder
import (
"io"
"io/ioutil"
"strings"
"time"
"unicode/utf8"
"code.cloudfoundry.org/clock"
"code.cloudfoundry.org/lager"
"github.com/concourse/concourse/atc"
"github.com/concourse/concourse/atc/db"
"github.com/concourse/concourse/atc/event"
"github.com/concourse/concourse/atc/exec"
"github.com/concourse/concourse/vars"
)
func NewDelegateFactory() *delegateFactory {
@ -20,29 +23,29 @@ func NewDelegateFactory() *delegateFactory {
type delegateFactory struct{}
func (delegate *delegateFactory) GetDelegate(build db.Build, planID atc.PlanID) exec.GetDelegate {
return NewGetDelegate(build, planID, clock.NewClock())
func (delegate *delegateFactory) GetDelegate(build db.Build, planID atc.PlanID, credVarsTracker vars.CredVarsTracker) exec.GetDelegate {
return NewGetDelegate(build, planID, credVarsTracker, clock.NewClock())
}
func (delegate *delegateFactory) PutDelegate(build db.Build, planID atc.PlanID) exec.PutDelegate {
return NewPutDelegate(build, planID, clock.NewClock())
func (delegate *delegateFactory) PutDelegate(build db.Build, planID atc.PlanID, credVarsTracker vars.CredVarsTracker) exec.PutDelegate {
return NewPutDelegate(build, planID, credVarsTracker, clock.NewClock())
}
func (delegate *delegateFactory) TaskDelegate(build db.Build, planID atc.PlanID) exec.TaskDelegate {
return NewTaskDelegate(build, planID, clock.NewClock())
func (delegate *delegateFactory) TaskDelegate(build db.Build, planID atc.PlanID, credVarsTracker vars.CredVarsTracker) exec.TaskDelegate {
return NewTaskDelegate(build, planID, credVarsTracker, clock.NewClock())
}
func (delegate *delegateFactory) CheckDelegate(check db.Check, planID atc.PlanID) exec.CheckDelegate {
return NewCheckDelegate(check, planID, clock.NewClock())
}
func (delegate *delegateFactory) BuildStepDelegate(build db.Build, planID atc.PlanID) exec.BuildStepDelegate {
return NewBuildStepDelegate(build, planID, clock.NewClock())
func (delegate *delegateFactory) BuildStepDelegate(build db.Build, planID atc.PlanID, credVarsTracker vars.CredVarsTracker) exec.BuildStepDelegate {
return NewBuildStepDelegate(build, planID, credVarsTracker, clock.NewClock())
}
func NewGetDelegate(build db.Build, planID atc.PlanID, clock clock.Clock) exec.GetDelegate {
func NewGetDelegate(build db.Build, planID atc.PlanID, credVarsTracker vars.CredVarsTracker, clock clock.Clock) exec.GetDelegate {
return &getDelegate{
BuildStepDelegate: NewBuildStepDelegate(build, planID, clock),
BuildStepDelegate: NewBuildStepDelegate(build, planID, credVarsTracker, clock),
eventOrigin: event.Origin{ID: event.OriginID(planID)},
build: build,
@ -138,9 +141,9 @@ func (d *getDelegate) UpdateVersion(log lager.Logger, plan atc.GetPlan, info exe
}
}
func NewPutDelegate(build db.Build, planID atc.PlanID, clock clock.Clock) exec.PutDelegate {
func NewPutDelegate(build db.Build, planID atc.PlanID, credVarsTracker vars.CredVarsTracker, clock clock.Clock) exec.PutDelegate {
return &putDelegate{
BuildStepDelegate: NewBuildStepDelegate(build, planID, clock),
BuildStepDelegate: NewBuildStepDelegate(build, planID, credVarsTracker, clock),
eventOrigin: event.Origin{ID: event.OriginID(planID)},
build: build,
@ -221,9 +224,9 @@ func (d *putDelegate) SaveOutput(log lager.Logger, plan atc.PutPlan, source atc.
}
}
func NewTaskDelegate(build db.Build, planID atc.PlanID, clock clock.Clock) exec.TaskDelegate {
func NewTaskDelegate(build db.Build, planID atc.PlanID, credVarsTracker vars.CredVarsTracker, clock clock.Clock) exec.TaskDelegate {
return &taskDelegate{
BuildStepDelegate: NewBuildStepDelegate(build, planID, clock),
BuildStepDelegate: NewBuildStepDelegate(build, planID, credVarsTracker, clock),
eventOrigin: event.Origin{ID: event.OriginID(planID)},
build: build,
@ -288,6 +291,8 @@ func NewCheckDelegate(check db.Check, planID atc.PlanID, clock clock.Clock) exec
}
type checkDelegate struct {
exec.BuildStepDelegate
check db.Check
eventOrigin event.Origin
clock clock.Clock
@ -305,25 +310,46 @@ func (*checkDelegate) Errored(lager.Logger, string) { retur
func NewBuildStepDelegate(
build db.Build,
planID atc.PlanID,
credVarsTracker vars.CredVarsTracker,
clock clock.Clock,
) *buildStepDelegate {
return &buildStepDelegate{
build: build,
planID: planID,
clock: clock,
build: build,
planID: planID,
clock: clock,
credVarsTracker: credVarsTracker,
}
}
type buildStepDelegate struct {
build db.Build
planID atc.PlanID
clock clock.Clock
build db.Build
planID atc.PlanID
clock clock.Clock
credVarsTracker vars.CredVarsTracker
}
func (delegate *buildStepDelegate) Variables() vars.CredVarsTracker {
return delegate.credVarsTracker
}
func (delegate *buildStepDelegate) ImageVersionDetermined(resourceCache db.UsedResourceCache) error {
return delegate.build.SaveImageResourceVersion(resourceCache)
}
type credVarsIterator struct {
line string
}
func (it *credVarsIterator) YieldCred(k, v string) {
it.line = strings.Replace(it.line, v, "[**redacted**]", -1)
}
func (delegate *buildStepDelegate) buildOutputFilter(str string) string {
it := &credVarsIterator{line: str}
delegate.credVarsTracker.IterateInterpolatedCreds(it)
return it.line
}
func (delegate *buildStepDelegate) Stdout() io.Writer {
return newDBEventWriter(
delegate.build,
@ -332,6 +358,7 @@ func (delegate *buildStepDelegate) Stdout() io.Writer {
ID: event.OriginID(delegate.planID),
},
delegate.clock,
delegate.buildOutputFilter,
)
}
@ -343,6 +370,7 @@ func (delegate *buildStepDelegate) Stderr() io.Writer {
ID: event.OriginID(delegate.planID),
},
delegate.clock,
delegate.buildOutputFilter,
)
}
@ -359,11 +387,12 @@ func (delegate *buildStepDelegate) Errored(logger lager.Logger, message string)
}
}
func newDBEventWriter(build db.Build, origin event.Origin, clock clock.Clock) io.Writer {
func newDBEventWriter(build db.Build, origin event.Origin, clock clock.Clock, filter exec.BuildOutputFilter) io.Writer {
return &dbEventWriter{
build: build,
origin: origin,
clock: clock,
filter: filter,
}
}
@ -372,6 +401,7 @@ type dbEventWriter struct {
origin event.Origin
clock clock.Clock
dangling []byte
filter exec.BuildOutputFilter
}
func (writer *dbEventWriter) Write(data []byte) (int, error) {
@ -385,9 +415,14 @@ func (writer *dbEventWriter) Write(data []byte) (int, error) {
writer.dangling = nil
payload := string(text)
if writer.filter != nil {
payload = writer.filter(payload)
}
err := writer.build.SaveEvent(event.Log{
Time: writer.clock.Now().Unix(),
Payload: string(text),
Payload: payload,
Origin: writer.origin,
})
if err != nil {

83
atc/engine/builder/delegate_factory_test.go
View File

@ -5,6 +5,9 @@ import (
"io"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"code.cloudfoundry.org/clock/fakeclock"
"code.cloudfoundry.org/lager"
"code.cloudfoundry.org/lager/lagertest"
@ -14,18 +17,17 @@ import (
"github.com/concourse/concourse/atc/engine/builder"
"github.com/concourse/concourse/atc/event"
"github.com/concourse/concourse/atc/exec"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/concourse/concourse/vars"
)
var _ = Describe("DelegateFactory", func() {
var (
logger *lagertest.TestLogger
fakeBuild *dbfakes.FakeBuild
fakePipeline *dbfakes.FakePipeline
fakeResource *dbfakes.FakeResource
fakeClock *fakeclock.FakeClock
logger *lagertest.TestLogger
fakeBuild *dbfakes.FakeBuild
fakePipeline *dbfakes.FakePipeline
fakeResource *dbfakes.FakeResource
fakeClock *fakeclock.FakeClock
credVarsTracker vars.CredVarsTracker
)
BeforeEach(func() {
@ -35,6 +37,8 @@ var _ = Describe("DelegateFactory", func() {
fakePipeline = new(dbfakes.FakePipeline)
fakeResource = new(dbfakes.FakeResource)
fakeClock = fakeclock.NewFakeClock(time.Unix(123456789, 0))
credVars := vars.StaticVariables{"source-param": "super-secret-source"}
credVarsTracker = vars.NewCredVarsTracker(credVars, true)
})
Describe("GetDelegate", func() {
@ -50,7 +54,7 @@ var _ = Describe("DelegateFactory", func() {
Metadata: []atc.MetadataField{{Name: "baz", Value: "shmaz"}},
}
delegate = builder.NewGetDelegate(fakeBuild, "some-plan-id", fakeClock)
delegate = builder.NewGetDelegate(fakeBuild, "some-plan-id", credVarsTracker, fakeClock)
})
Describe("Finished", func() {
@ -160,7 +164,7 @@ var _ = Describe("DelegateFactory", func() {
Metadata: []atc.MetadataField{{Name: "baz", Value: "shmaz"}},
}
delegate = builder.NewPutDelegate(fakeBuild, "some-plan-id", fakeClock)
delegate = builder.NewPutDelegate(fakeBuild, "some-plan-id", credVarsTracker, fakeClock)
})
Describe("Finished", func() {
@ -219,7 +223,7 @@ var _ = Describe("DelegateFactory", func() {
)
BeforeEach(func() {
delegate = builder.NewTaskDelegate(fakeBuild, "some-plan-id", fakeClock)
delegate = builder.NewTaskDelegate(fakeBuild, "some-plan-id", credVarsTracker, fakeClock)
})
Describe("Initializing", func() {
@ -292,7 +296,7 @@ var _ = Describe("DelegateFactory", func() {
)
BeforeEach(func() {
delegate = builder.NewBuildStepDelegate(fakeBuild, "some-plan-id", fakeClock)
delegate = builder.NewBuildStepDelegate(fakeBuild, "some-plan-id", credVarsTracker, fakeClock)
})
Describe("ImageVersionDetermined", func() {
@ -456,5 +460,60 @@ var _ = Describe("DelegateFactory", func() {
})
})
})
Describe("Secrets redacting", func() {
var (
writer io.Writer
writtenBytes int
writeErr error
)
BeforeEach(func() {
delegate.Variables().Get(vars.VariableDefinition{Name: "source-param"})
})
Context("Stdout", func() {
JustBeforeEach(func() {
writer = delegate.Stdout()
writtenBytes, writeErr = writer.Write([]byte("ok super-secret-source ok"))
})
It("should be redacted", func() {
Expect(writeErr).To(BeNil())
Expect(writtenBytes).To(Equal(len("ok super-secret-source ok")))
Expect(fakeBuild.SaveEventCallCount()).To(Equal(1))
Expect(fakeBuild.SaveEventArgsForCall(0)).To(Equal(event.Log{
Time: 123456789,
Payload: "ok [**redacted**] ok",
Origin: event.Origin{
Source: event.OriginSourceStdout,
ID: "some-plan-id",
},
}))
})
})
Context("Stderr", func() {
JustBeforeEach(func() {
writer = delegate.Stderr()
writtenBytes, writeErr = writer.Write([]byte("ok super-secret-source ok"))
})
It("should be redacted", func() {
Expect(writeErr).To(BeNil())
Expect(writtenBytes).To(Equal(len("ok super-secret-source ok")))
Expect(fakeBuild.SaveEventCallCount()).To(Equal(1))
Expect(fakeBuild.SaveEventArgsForCall(0)).To(Equal(event.Log{
Time: 123456789,
Payload: "ok [**redacted**] ok",
Origin: event.Origin{
Source: event.OriginSourceStderr,
ID: "some-plan-id",
},
}))
})
})
})
})
})

8
atc/engine/builder/step_factory.go
View File

@ -6,7 +6,6 @@ import (
"path/filepath"
"github.com/concourse/concourse/atc"
"github.com/concourse/concourse/atc/creds"
"github.com/concourse/concourse/atc/db"
"github.com/concourse/concourse/atc/db/lock"
"github.com/concourse/concourse/atc/exec"
@ -21,7 +20,6 @@ type stepFactory struct {
resourceFetcher fetcher.Fetcher
resourceCacheFactory db.ResourceCacheFactory
resourceConfigFactory db.ResourceConfigFactory
secretManager creds.Secrets
defaultLimits atc.ContainerLimits
strategy worker.ContainerPlacementStrategy
resourceFactory resource.ResourceFactory
@ -34,7 +32,6 @@ func NewStepFactory(
resourceFetcher fetcher.Fetcher,
resourceCacheFactory db.ResourceCacheFactory,
resourceConfigFactory db.ResourceConfigFactory,
secretManager creds.Secrets,
defaultLimits atc.ContainerLimits,
strategy worker.ContainerPlacementStrategy,
resourceFactory resource.ResourceFactory,
@ -46,7 +43,6 @@ func NewStepFactory(
resourceFetcher: resourceFetcher,
resourceCacheFactory: resourceCacheFactory,
resourceConfigFactory: resourceConfigFactory,
secretManager: secretManager,
defaultLimits: defaultLimits,
strategy: strategy,
resourceFactory: resourceFactory,
@ -67,7 +63,6 @@ func (factory *stepFactory) GetStep(
*plan.Get,
stepMetadata,
containerMetadata,
factory.secretManager,
factory.resourceFetcher,
factory.resourceCacheFactory,
factory.strategy,
@ -91,7 +86,6 @@ func (factory *stepFactory) PutStep(
*plan.Put,
stepMetadata,
containerMetadata,
factory.secretManager,
factory.resourceFactory,
factory.resourceConfigFactory,
factory.strategy,
@ -115,7 +109,6 @@ func (factory *stepFactory) CheckStep(
*plan.Check,
stepMetadata,
containerMetadata,
factory.secretManager,
factory.resourceFactory,
worker.NewRandomPlacementStrategy(),
factory.pool,
@ -140,7 +133,6 @@ func (factory *stepFactory) TaskStep(
factory.defaultLimits,
stepMetadata,
containerMetadata,
factory.secretManager,
factory.strategy,
factory.client,
delegate,

9
atc/engine/engine.go
View File

@ -8,7 +8,9 @@ import (
"code.cloudfoundry.org/lager"
"code.cloudfoundry.org/lager/lagerctx"
"github.com/concourse/concourse/atc"
"github.com/concourse/concourse/atc/creds"
"github.com/concourse/concourse/atc/db"
"github.com/concourse/concourse/atc/exec"
"github.com/concourse/concourse/atc/metric"
@ -37,8 +39,7 @@ type StepBuilder interface {
func NewEngine(builder StepBuilder) Engine {
return &engine{
builder: builder,
builder: builder,
release: make(chan bool),
trackedStates: new(sync.Map),
waitGroup: new(sync.WaitGroup),
@ -46,7 +47,9 @@ func NewEngine(builder StepBuilder) Engine {
}
type engine struct {
builder StepBuilder
builder StepBuilder
secrets creds.Secrets
redactSecrets bool
release chan bool
trackedStates *sync.Map

6
atc/exec/check_step.go
View File

@ -9,7 +9,6 @@ import (
"code.cloudfoundry.org/lager"
"code.cloudfoundry.org/lager/lagerctx"
"github.com/concourse/concourse/atc"
"github.com/concourse/concourse/atc/creds"
"github.com/concourse/concourse/atc/db"
"github.com/concourse/concourse/atc/metric"
"github.com/concourse/concourse/atc/resource"
@ -21,7 +20,6 @@ type CheckStep struct {
plan atc.CheckPlan
metadata StepMetadata
containerMetadata db.ContainerMetadata
secrets creds.Secrets
resourceFactory resource.ResourceFactory
strategy worker.ContainerPlacementStrategy
pool worker.Pool
@ -29,6 +27,8 @@ type CheckStep struct {
succeeded bool
}
//go:generate counterfeiter . CheckDelegate
type CheckDelegate interface {
BuildStepDelegate
@ -40,7 +40,6 @@ func NewCheckStep(
plan atc.CheckPlan,
metadata StepMetadata,
containerMetadata db.ContainerMetadata,
secrets creds.Secrets,
resourceFactory resource.ResourceFactory,
strategy worker.ContainerPlacementStrategy,
pool worker.Pool,
@ -50,7 +49,6 @@ func NewCheckStep(
planID: planID,
plan: plan,
metadata: metadata,
secrets: secrets,
containerMetadata: containerMetadata,
resourceFactory: resourceFactory,
pool: pool,

1
atc/exec/check_step_test.go
View File

@ -115,7 +115,6 @@ var _ = Describe("CheckStep", func() {
*plan.Check,
stepMetadata,
containerMetadata,
fakeSecretManager,
fakeResourceFactory,
fakeStrategy,
fakePool,

65
atc/exec/execfakes/fake_build_step_delegate.go
View File

@ -8,6 +8,7 @@ import (
"code.cloudfoundry.org/lager"
"github.com/concourse/concourse/atc/db"
"github.com/concourse/concourse/atc/exec"
"github.com/concourse/concourse/vars"
)
type FakeBuildStepDelegate struct {
@ -48,6 +49,16 @@ type FakeBuildStepDelegate struct {
stdoutReturnsOnCall map[int]struct {
result1 io.Writer
}
VariablesStub func() vars.CredVarsTracker
variablesMutex sync.RWMutex
variablesArgsForCall []struct {
}
variablesReturns struct {
result1 vars.CredVarsTracker
}
variablesReturnsOnCall map[int]struct {
result1 vars.CredVarsTracker
}
invocations map[string][][]interface{}
invocationsMutex sync.RWMutex
}
@ -248,6 +259,58 @@ func (fake *FakeBuildStepDelegate) StdoutReturnsOnCall(i int, result1 io.Writer)
}{result1}
}
func (fake *FakeBuildStepDelegate) Variables() vars.CredVarsTracker {
fake.variablesMutex.Lock()
ret, specificReturn := fake.variablesReturnsOnCall[len(fake.variablesArgsForCall)]
fake.variablesArgsForCall = append(fake.variablesArgsForCall, struct {
}{})
fake.recordInvocation("Variables", []interface{}{})
fake.variablesMutex.Unlock()
if fake.VariablesStub != nil {
return fake.VariablesStub()
}
if specificReturn {
return ret.result1
}
fakeReturns := fake.variablesReturns
return fakeReturns.result1
}
func (fake *FakeBuildStepDelegate) VariablesCallCount() int {
fake.variablesMutex.RLock()
defer fake.variablesMutex.RUnlock()
return len(fake.variablesArgsForCall)
}
func (fake *FakeBuildStepDelegate) VariablesCalls(stub func() vars.CredVarsTracker) {
fake.variablesMutex.Lock()
defer fake.variablesMutex.Unlock()
fake.VariablesStub = stub
}
func (fake *FakeBuildStepDelegate) VariablesReturns(result1 vars.CredVarsTracker) {
fake.variablesMutex.Lock()
defer fake.variablesMutex.Unlock()
fake.VariablesStub = nil
fake.variablesReturns = struct {
result1 vars.CredVarsTracker
}{result1}
}
func (fake *FakeBuildStepDelegate) VariablesReturnsOnCall(i int, result1 vars.CredVarsTracker) {
fake.variablesMutex.Lock()
defer fake.variablesMutex.Unlock()
fake.VariablesStub = nil
if fake.variablesReturnsOnCall == nil {
fake.variablesReturnsOnCall = make(map[int]struct {
result1 vars.CredVarsTracker
})
}
fake.variablesReturnsOnCall[i] = struct {
result1 vars.CredVarsTracker
}{result1}
}
func (fake *FakeBuildStepDelegate) Invocations() map[string][][]interface{} {
fake.invocationsMutex.RLock()
defer fake.invocationsMutex.RUnlock()
@ -259,6 +322,8 @@ func (fake *FakeBuildStepDelegate) Invocations() map[string][][]interface{} {
defer fake.stderrMutex.RUnlock()
fake.stdoutMutex.RLock()
defer fake.stdoutMutex.RUnlock()
fake.variablesMutex.RLock()
defer fake.variablesMutex.RUnlock()
copiedInvocations := map[string][][]interface{}{}
for key, value := range fake.invocations {
copiedInvocations[key] = value

65
atc/exec/execfakes/fake_check_delegate.go
View File

@ -9,6 +9,7 @@ import (
"github.com/concourse/concourse/atc"
"github.com/concourse/concourse/atc/db"
"github.com/concourse/concourse/atc/exec"
"github.com/concourse/concourse/vars"
)
type FakeCheckDelegate struct {
@ -60,6 +61,16 @@ type FakeCheckDelegate struct {
stdoutReturnsOnCall map[int]struct {
result1 io.Writer
}
VariablesStub func() vars.CredVarsTracker
variablesMutex sync.RWMutex
variablesArgsForCall []struct {
}
variablesReturns struct {
result1 vars.CredVarsTracker
}
variablesReturnsOnCall map[int]struct {
result1 vars.CredVarsTracker
}
invocations map[string][][]interface{}
invocationsMutex sync.RWMutex
}
@ -325,6 +336,58 @@ func (fake *FakeCheckDelegate) StdoutReturnsOnCall(i int, result1 io.Writer) {
}{result1}
}
func (fake *FakeCheckDelegate) Variables() vars.CredVarsTracker {
fake.variablesMutex.Lock()
ret, specificReturn := fake.variablesReturnsOnCall[len(fake.variablesArgsForCall)]
fake.variablesArgsForCall = append(fake.variablesArgsForCall, struct {
}{})
fake.recordInvocation("Variables", []interface{}{})
fake.variablesMutex.Unlock()
if fake.VariablesStub != nil {
return fake.VariablesStub()
}
if specificReturn {
return ret.result1
}
fakeReturns := fake.variablesReturns
return fakeReturns.result1
}
func (fake *FakeCheckDelegate) VariablesCallCount() int {
fake.variablesMutex.RLock()
defer fake.variablesMutex.RUnlock()
return len(fake.variablesArgsForCall)
}
func (fake *FakeCheckDelegate) VariablesCalls(stub func() vars.CredVarsTracker) {
fake.variablesMutex.Lock()
defer fake.variablesMutex.Unlock()
fake.VariablesStub = stub
}
func (fake *FakeCheckDelegate) VariablesReturns(result1 vars.CredVarsTracker) {
fake.variablesMutex.Lock()
defer fake.variablesMutex.Unlock()
fake.VariablesStub = nil
fake.variablesReturns = struct {
result1 vars.CredVarsTracker
}{result1}
}
func (fake *FakeCheckDelegate) VariablesReturnsOnCall(i int, result1 vars.CredVarsTracker) {
fake.variablesMutex.Lock()
defer fake.variablesMutex.Unlock()
fake.VariablesStub = nil
if fake.variablesReturnsOnCall == nil {
fake.variablesReturnsOnCall = make(map[int]struct {
result1 vars.CredVarsTracker
})
}
fake.variablesReturnsOnCall[i] = struct {
result1 vars.CredVarsTracker
}{result1}
}
func (fake *FakeCheckDelegate) Invocations() map[string][][]interface{} {
fake.invocationsMutex.RLock()
defer fake.invocationsMutex.RUnlock()
@ -338,6 +401,8 @@ func (fake *FakeCheckDelegate) Invocations() map[string][][]interface{} {
defer fake.stderrMutex.RUnlock()
fake.stdoutMutex.RLock()
defer fake.stdoutMutex.RUnlock()
fake.variablesMutex.RLock()
defer fake.variablesMutex.RUnlock()
copiedInvocations := map[string][][]interface{}{}
for key, value := range fake.invocations {
copiedInvocations[key] = value

65
atc/exec/execfakes/fake_get_delegate.go
View File

@ -9,6 +9,7 @@ import (
"github.com/concourse/concourse/atc"
"github.com/concourse/concourse/atc/db"
"github.com/concourse/concourse/atc/exec"
"github.com/concourse/concourse/vars"
)
type FakeGetDelegate struct {
@ -73,6 +74,16 @@ type FakeGetDelegate struct {
arg2 atc.GetPlan
arg3 exec.VersionInfo
}
VariablesStub func() vars.CredVarsTracker
variablesMutex sync.RWMutex
variablesArgsForCall []struct {
}
variablesReturns struct {
result1 vars.CredVarsTracker
}
variablesReturnsOnCall map[int]struct {
result1 vars.CredVarsTracker
}
invocations map[string][][]interface{}
invocationsMutex sync.RWMutex
}
@ -401,6 +412,58 @@ func (fake *FakeGetDelegate) UpdateVersionArgsForCall(i int) (lager.Logger, atc.
return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3
}
func (fake *FakeGetDelegate) Variables() vars.CredVarsTracker {
fake.variablesMutex.Lock()
ret, specificReturn := fake.variablesReturnsOnCall[len(fake.variablesArgsForCall)]
fake.variablesArgsForCall = append(fake.variablesArgsForCall, struct {
}{})
fake.recordInvocation("Variables", []interface{}{})
fake.variablesMutex.Unlock()
if fake.VariablesStub != nil {
return fake.VariablesStub()
}
if specificReturn {
return ret.result1
}
fakeReturns := fake.variablesReturns
return fakeReturns.result1
}
func (fake *FakeGetDelegate) VariablesCallCount() int {
fake.variablesMutex.RLock()
defer fake.variablesMutex.RUnlock()
return len(fake.variablesArgsForCall)
}
func (fake *FakeGetDelegate) VariablesCalls(stub func() vars.CredVarsTracker) {
fake.variablesMutex.Lock()
defer fake.variablesMutex.Unlock()
fake.VariablesStub = stub
}
func (fake *FakeGetDelegate) VariablesReturns(result1 vars.CredVarsTracker) {
fake.variablesMutex.Lock()
defer fake.variablesMutex.Unlock()
fake.VariablesStub = nil
fake.variablesReturns = struct {
result1 vars.CredVarsTracker
}{result1}
}
func (fake *FakeGetDelegate) VariablesReturnsOnCall(i int, result1 vars.CredVarsTracker) {
fake.variablesMutex.Lock()
defer fake.variablesMutex.Unlock()
fake.VariablesStub = nil
if fake.variablesReturnsOnCall == nil {
fake.variablesReturnsOnCall = make(map[int]struct {
result1 vars.CredVarsTracker
})
}
fake.variablesReturnsOnCall[i] = struct {
result1 vars.CredVarsTracker
}{result1}
}
func (fake *FakeGetDelegate) Invocations() map[string][][]interface{} {
fake.invocationsMutex.RLock()
defer fake.invocationsMutex.RUnlock()
@ -420,6 +483,8 @@ func (fake *FakeGetDelegate) Invocations() map[string][][]interface{} {
defer fake.stdoutMutex.RUnlock()
fake.updateVersionMutex.RLock()
defer fake.updateVersionMutex.RUnlock()
fake.variablesMutex.RLock()
defer fake.variablesMutex.RUnlock()
copiedInvocations := map[string][][]interface{}{}
for key, value := range fake.invocations {
copiedInvocations[key] = value

65
atc/exec/execfakes/fake_put_delegate.go
View File

@ -9,6 +9,7 @@ import (
"github.com/concourse/concourse/atc"
"github.com/concourse/concourse/atc/db"
"github.com/concourse/concourse/atc/exec"
"github.com/concourse/concourse/vars"
)
type FakePutDelegate struct {
@ -75,6 +76,16 @@ type FakePutDelegate struct {
stdoutReturnsOnCall map[int]struct {
result1 io.Writer
}
VariablesStub func() vars.CredVarsTracker
variablesMutex sync.RWMutex
variablesArgsForCall []struct {
}
variablesReturns struct {
result1 vars.CredVarsTracker
}
variablesReturnsOnCall map[int]struct {
result1 vars.CredVarsTracker
}
invocations map[string][][]interface{}
invocationsMutex sync.RWMutex
}
@ -405,6 +416,58 @@ func (fake *FakePutDelegate) StdoutReturnsOnCall(i int, result1 io.Writer) {
}{result1}
}
func (fake *FakePutDelegate) Variables() vars.CredVarsTracker {
fake.variablesMutex.Lock()
ret, specificReturn := fake.variablesReturnsOnCall[len(fake.variablesArgsForCall)]
fake.variablesArgsForCall = append(fake.variablesArgsForCall, struct {
}{})
fake.recordInvocation("Variables", []interface{}{})
fake.variablesMutex.Unlock()
if fake.VariablesStub != nil {
return fake.VariablesStub()
}
if specificReturn {
return ret.result1
}
fakeReturns := fake.variablesReturns
return fakeReturns.result1
}
func (fake *FakePutDelegate) VariablesCallCount() int {
fake.variablesMutex.RLock()
defer fake.variablesMutex.RUnlock()
return len(fake.variablesArgsForCall)
}
func (fake *FakePutDelegate) VariablesCalls(stub func() vars.CredVarsTracker) {
fake.variablesMutex.Lock()
defer fake.variablesMutex.Unlock()
fake.VariablesStub = stub
}
func (fake *FakePutDelegate) VariablesReturns(result1 vars.CredVarsTracker) {
fake.variablesMutex.Lock()
defer fake.variablesMutex.Unlock()
fake.VariablesStub = nil
fake.variablesReturns = struct {
result1 vars.CredVarsTracker
}{result1}
}
func (fake *FakePutDelegate) VariablesReturnsOnCall(i int, result1 vars.CredVarsTracker) {
fake.variablesMutex.Lock()
defer fake.variablesMutex.Unlock()
fake.VariablesStub = nil
if fake.variablesReturnsOnCall == nil {
fake.variablesReturnsOnCall = make(map[int]struct {
result1 vars.CredVarsTracker
})
}
fake.variablesReturnsOnCall[i] = struct {
result1 vars.CredVarsTracker
}{result1}
}
func (fake *FakePutDelegate) Invocations() map[string][][]interface{} {
fake.invocationsMutex.RLock()
defer fake.invocationsMutex.RUnlock()
@ -424,6 +487,8 @@ func (fake *FakePutDelegate) Invocations() map[string][][]interface{} {
defer fake.stderrMutex.RUnlock()
fake.stdoutMutex.RLock()
defer fake.stdoutMutex.RUnlock()
fake.variablesMutex.RLock()
defer fake.variablesMutex.RUnlock()
copiedInvocations := map[string][][]interface{}{}
for key, value := range fake.invocations {
copiedInvocations[key] = value

65
atc/exec/execfakes/fake_task_delegate.go
View File

@ -9,6 +9,7 @@ import (
"github.com/concourse/concourse/atc"
"github.com/concourse/concourse/atc/db"
"github.com/concourse/concourse/atc/exec"
"github.com/concourse/concourse/vars"
)
type FakeTaskDelegate struct {
@ -67,6 +68,16 @@ type FakeTaskDelegate struct {
stdoutReturnsOnCall map[int]struct {
result1 io.Writer
}
VariablesStub func() vars.CredVarsTracker
variablesMutex sync.RWMutex
variablesArgsForCall []struct {
}
variablesReturns struct {
result1 vars.CredVarsTracker
}
variablesReturnsOnCall map[int]struct {
result1 vars.CredVarsTracker
}
invocations map[string][][]interface{}
invocationsMutex sync.RWMutex
}
@ -363,6 +374,58 @@ func (fake *FakeTaskDelegate) StdoutReturnsOnCall(i int, result1 io.Writer) {
}{result1}
}
func (fake *FakeTaskDelegate) Variables() vars.CredVarsTracker {
fake.variablesMutex.Lock()
ret, specificReturn := fake.variablesReturnsOnCall[len(fake.variablesArgsForCall)]
fake.variablesArgsForCall = append(fake.variablesArgsForCall, struct {
}{})
fake.recordInvocation("Variables", []interface{}{})
fake.variablesMutex.Unlock()
if fake.VariablesStub != nil {
return fake.VariablesStub()
}
if specificReturn {
return ret.result1
}
fakeReturns := fake.variablesReturns
return fakeReturns.result1
}
func (fake *FakeTaskDelegate) VariablesCallCount() int {
fake.variablesMutex.RLock()
defer fake.variablesMutex.RUnlock()
return len(fake.variablesArgsForCall)
}
func (fake *FakeTaskDelegate) VariablesCalls(stub func() vars.CredVarsTracker) {
fake.variablesMutex.Lock()
defer fake.variablesMutex.Unlock()
fake.VariablesStub = stub
}
func (fake *FakeTaskDelegate) VariablesReturns(result1 vars.CredVarsTracker) {
fake.variablesMutex.Lock()
defer fake.variablesMutex.Unlock()
fake.VariablesStub = nil
fake.variablesReturns = struct {
result1 vars.CredVarsTracker
}{result1}
}
func (fake *FakeTaskDelegate) VariablesReturnsOnCall(i int, result1 vars.CredVarsTracker) {
fake.variablesMutex.Lock()
defer fake.variablesMutex.Unlock()
fake.VariablesStub = nil
if fake.variablesReturnsOnCall == nil {
fake.variablesReturnsOnCall = make(map[int]struct {
result1 vars.CredVarsTracker
})
}
fake.variablesReturnsOnCall[i] = struct {
result1 vars.CredVarsTracker
}{result1}
}
func (fake *FakeTaskDelegate) Invocations() map[string][][]interface{} {
fake.invocationsMutex.RLock()
defer fake.invocationsMutex.RUnlock()
@ -380,6 +443,8 @@ func (fake *FakeTaskDelegate) Invocations() map[string][][]interface{} {
defer fake.stderrMutex.RUnlock()
fake.stdoutMutex.RLock()
defer fake.stdoutMutex.RUnlock()
fake.variablesMutex.RLock()
defer fake.variablesMutex.RUnlock()
copiedInvocations := map[string][][]interface{}{}
for key, value := range fake.invocations {
copiedInvocations[key] = value

5
atc/exec/get_step.go
View File

@ -54,7 +54,6 @@ type GetStep struct {
plan atc.GetPlan
metadata StepMetadata
containerMetadata db.ContainerMetadata
secrets creds.Secrets
resourceFetcher fetcher.Fetcher
resourceCacheFactory db.ResourceCacheFactory
strategy worker.ContainerPlacementStrategy
@ -68,7 +67,6 @@ func NewGetStep(
plan atc.GetPlan,
metadata StepMetadata,
containerMetadata db.ContainerMetadata,
secrets creds.Secrets,
resourceFetcher fetcher.Fetcher,
resourceCacheFactory db.ResourceCacheFactory,
strategy worker.ContainerPlacementStrategy,
@ -80,7 +78,6 @@ func NewGetStep(
plan: plan,
metadata: metadata,
containerMetadata: containerMetadata,
secrets: secrets,
resourceFetcher: resourceFetcher,
resourceCacheFactory: resourceCacheFactory,
strategy: strategy,
@ -121,7 +118,7 @@ func (step *GetStep) Run(ctx context.Context, state RunState) error {
step.delegate.Initializing(logger)
variables := creds.NewVariables(step.secrets, step.metadata.TeamName, step.metadata.PipelineName)
variables := step.delegate.Variables()
source, err := creds.NewSource(variables, step.plan.Source).Evaluate()
if err != nil {

35
atc/exec/get_step_test.go
View File

@ -9,10 +9,13 @@ import (
"io"
"io/ioutil"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gbytes"
"code.cloudfoundry.org/lager/lagertest"
"github.com/DataDog/zstd"
"github.com/concourse/concourse/atc"
"github.com/concourse/concourse/atc/creds/credsfakes"
"github.com/concourse/concourse/atc/db"
"github.com/concourse/concourse/atc/db/dbfakes"
"github.com/concourse/concourse/atc/exec"
@ -23,9 +26,7 @@ import (
"github.com/concourse/concourse/atc/resource/resourcefakes"
"github.com/concourse/concourse/atc/worker"
"github.com/concourse/concourse/atc/worker/workerfakes"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gbytes"
"github.com/concourse/concourse/vars"
)
var _ = Describe("GetStep", func() {
@ -39,7 +40,6 @@ var _ = Describe("GetStep", func() {
fakeStrategy *workerfakes.FakeContainerPlacementStrategy
fakeResourceFetcher *fetcherfakes.FakeFetcher
fakeResourceCacheFactory *dbfakes.FakeResourceCacheFactory
fakeSecretManager *credsfakes.FakeSecrets
fakeDelegate *execfakes.FakeGetDelegate
getPlan *atc.GetPlan
@ -52,6 +52,8 @@ var _ = Describe("GetStep", func() {
getStep exec.Step
stepErr error
credVarsTracker vars.CredVarsTracker
containerMetadata = db.ContainerMetadata{
WorkingDirectory: resource.ResourcesDir("get"),
PipelineID: 4567,
@ -81,8 +83,8 @@ var _ = Describe("GetStep", func() {
fakeStrategy = new(workerfakes.FakeContainerPlacementStrategy)
fakeResourceCacheFactory = new(dbfakes.FakeResourceCacheFactory)
fakeSecretManager = new(credsfakes.FakeSecrets)
fakeSecretManager.GetReturns("super-secret-source", nil, true, nil)
credVars := vars.StaticVariables{"source-param": "super-secret-source"}
credVarsTracker = vars.NewCredVarsTracker(credVars, true)
artifactRepository = artifact.NewRepository()
state = new(execfakes.FakeRunState)
@ -92,6 +94,7 @@ var _ = Describe("GetStep", func() {
fakeResourceFetcher.FetchReturns(fakeVersionedSource, nil)
fakeDelegate = new(execfakes.FakeGetDelegate)
fakeDelegate.VariablesReturns(credVarsTracker)
uninterpolatedResourceTypes := atc.VersionedResourceTypes{
{
@ -141,7 +144,6 @@ var _ = Describe("GetStep", func() {
*plan.Get,
stepMetadata,
containerMetadata,
fakeSecretManager,
fakeResourceFetcher,
fakeResourceCacheFactory,
fakeStrategy,
@ -184,12 +186,11 @@ var _ = Describe("GetStep", func() {
Expect(fakeResourceFetcher.FetchCallCount()).To(Equal(1))
fctx, _, actualContainerMetadata, actualWorker, actualContainerSpec, actualResourceTypes, resourceInstance, delegate := fakeResourceFetcher.FetchArgsForCall(0)
Expect(fctx).To(Equal(ctx))
Expect(actualContainerMetadata).To(Equal(
db.ContainerMetadata{
PipelineID: 4567,
Type: db.ContainerTypeGet,
StepName: "some-step",
WorkingDirectory: "/tmp/build/get",
Expect(actualContainerMetadata).To(Equal(db.ContainerMetadata{
PipelineID: 4567,
Type: db.ContainerTypeGet,
StepName: "some-step",
WorkingDirectory: "/tmp/build/get",
}))
Expect(actualWorker.Name()).To(Equal("some-worker"))
Expect(actualContainerSpec).To(Equal(worker.ContainerSpec{
@ -219,6 +220,12 @@ var _ = Describe("GetStep", func() {
Expect(resourceInstance.LockName("fake-worker")).To(Equal(expectedLockName))
})
It("secrets are tracked", func() {
mapit := vars.NewMapCredVarsTrackerIterator()
credVarsTracker.IterateInterpolatedCreds(mapit)
Expect(mapit.Data["source-param"]).To(Equal("super-secret-source"))
})
Context("when fetching resource succeeds", func() {
BeforeEach(func() {
fakeVersionedSource.VersionReturns(atc.Version{"some": "version"})

5
atc/exec/put_step.go
View File

@ -30,7 +30,6 @@ type PutStep struct {
plan atc.PutPlan
metadata StepMetadata
containerMetadata db.ContainerMetadata
secrets creds.Secrets
resourceFactory resource.ResourceFactory
resourceConfigFactory db.ResourceConfigFactory
strategy worker.ContainerPlacementStrategy
@ -44,7 +43,6 @@ func NewPutStep(
plan atc.PutPlan,
metadata StepMetadata,
containerMetadata db.ContainerMetadata,
secrets creds.Secrets,
resourceFactory resource.ResourceFactory,
resourceConfigFactory db.ResourceConfigFactory,
strategy worker.ContainerPlacementStrategy,
@ -56,7 +54,6 @@ func NewPutStep(
plan: plan,
metadata: metadata,
containerMetadata: containerMetadata,
secrets: secrets,
resourceFactory: resourceFactory,
resourceConfigFactory: resourceConfigFactory,
pool: pool,
@ -82,7 +79,7 @@ func (step *PutStep) Run(ctx context.Context, state RunState) error {
step.delegate.Initializing(logger)
variables := creds.NewVariables(step.secrets, step.metadata.TeamName, step.metadata.PipelineName)
variables := step.delegate.Variables()
source, err := creds.NewSource(variables, step.plan.Source).Evaluate()
if err != nil {

26
atc/exec/put_step_test.go
View File

@ -4,8 +4,11 @@ import (
"context"
"errors"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gbytes"
"github.com/concourse/concourse/atc"
"github.com/concourse/concourse/atc/creds/credsfakes"
"github.com/concourse/concourse/atc/db"
"github.com/concourse/concourse/atc/db/dbfakes"
"github.com/concourse/concourse/atc/exec"
@ -15,9 +18,7 @@ import (
"github.com/concourse/concourse/atc/resource/resourcefakes"
"github.com/concourse/concourse/atc/worker"
"github.com/concourse/concourse/atc/worker/workerfakes"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gbytes"
"github.com/concourse/concourse/vars"
)
var _ = Describe("PutStep", func() {
@ -30,7 +31,6 @@ var _ = Describe("PutStep", func() {
fakeStrategy *workerfakes.FakeContainerPlacementStrategy
fakeResourceFactory *resourcefakes.FakeResourceFactory
fakeResourceConfigFactory *dbfakes.FakeResourceConfigFactory
fakeSecretManager *credsfakes.FakeSecrets
fakeDelegate *execfakes.FakePutDelegate
putPlan *atc.PutPlan
@ -57,6 +57,8 @@ var _ = Describe("PutStep", func() {
putStep *exec.PutStep
stepErr error
credVarsTracker vars.CredVarsTracker
stdoutBuf *gbytes.Buffer
stderrBuf *gbytes.Buffer
@ -74,15 +76,15 @@ var _ = Describe("PutStep", func() {
fakeResourceFactory = new(resourcefakes.FakeResourceFactory)
fakeResourceConfigFactory = new(dbfakes.FakeResourceConfigFactory)
fakeSecretManager = new(credsfakes.FakeSecrets)
fakeSecretManager.GetReturnsOnCall(0, "super-secret-source", nil, true, nil)
fakeSecretManager.GetReturnsOnCall(1, "source", nil, true, nil)
credVars := vars.StaticVariables{"custom-param": "source", "source-param": "super-secret-source"}
credVarsTracker = vars.NewCredVarsTracker(credVars, true)
fakeDelegate = new(execfakes.FakePutDelegate)
stdoutBuf = gbytes.NewBuffer()
stderrBuf = gbytes.NewBuffer()
fakeDelegate.StdoutReturns(stdoutBuf)
fakeDelegate.StderrReturns(stderrBuf)
fakeDelegate.VariablesReturns(vars.NewCredVarsTracker(credVarsTracker, false))
repo = artifact.NewRepository()
state = new(execfakes.FakeRunState)
@ -136,7 +138,6 @@ var _ = Describe("PutStep", func() {
*plan.Put,
stepMetadata,
containerMetadata,
fakeSecretManager,
fakeResourceFactory,
fakeResourceConfigFactory,
fakeStrategy,
@ -235,6 +236,13 @@ var _ = Describe("PutStep", func() {
Expect(delegate).To(Equal(fakeDelegate))
})
It("secrets are tracked", func() {
mapit := vars.NewMapCredVarsTrackerIterator()
credVarsTracker.IterateInterpolatedCreds(mapit)
Expect(mapit.Data["custom-param"]).To(Equal("source"))
Expect(mapit.Data["source-param"]).To(Equal("super-secret-source"))
})
Context("when the inputs are specified", func() {
BeforeEach(func() {
putPlan.Inputs = &atc.InputsConfig{

5
atc/exec/step.go
View File

@ -5,6 +5,7 @@ import (
"io"
"code.cloudfoundry.org/lager"
"github.com/concourse/concourse/vars"
"github.com/concourse/concourse/atc"
"github.com/concourse/concourse/atc/db"
"github.com/concourse/concourse/atc/exec/artifact"
@ -33,12 +34,16 @@ type Step interface {
//go:generate counterfeiter . BuildStepDelegate
type BuildOutputFilter func(text string) string
type BuildStepDelegate interface {
ImageVersionDetermined(db.UsedResourceCache) error
Stdout() io.Writer
Stderr() io.Writer
Variables() vars.CredVarsTracker
Errored(lager.Logger, string)
}

5
atc/exec/task_step.go
View File

@ -68,7 +68,6 @@ type TaskStep struct {
defaultLimits atc.ContainerLimits
metadata StepMetadata
containerMetadata db.ContainerMetadata
secrets creds.Secrets
strategy worker.ContainerPlacementStrategy
workerClient worker.Client
delegate TaskDelegate
@ -82,7 +81,6 @@ func NewTaskStep(
defaultLimits atc.ContainerLimits,
metadata StepMetadata,
containerMetadata db.ContainerMetadata,
secrets creds.Secrets,
strategy worker.ContainerPlacementStrategy,
workerClient worker.Client,
delegate TaskDelegate,
@ -94,7 +92,6 @@ func NewTaskStep(
defaultLimits: defaultLimits,
metadata: metadata,
containerMetadata: containerMetadata,
secrets: secrets,
strategy: strategy,
workerClient: workerClient,
delegate: delegate,
@ -124,7 +121,7 @@ func (step *TaskStep) Run(ctx context.Context, state RunState) error {
"job-id": step.metadata.JobID,
})
variables := creds.NewVariables(step.secrets, step.metadata.TeamName, step.metadata.PipelineName)
variables := step.delegate.Variables()
resourceTypes, err := creds.NewVersionedResourceTypes(variables, step.plan.VersionedResourceTypes).Evaluate()
if err != nil {

21
atc/exec/task_step_test.go
View File

@ -16,7 +16,6 @@ import (
"github.com/onsi/gomega/gbytes"
"github.com/concourse/concourse/atc"
"github.com/concourse/concourse/atc/creds/credsfakes"
"github.com/concourse/concourse/atc/db"
"github.com/concourse/concourse/atc/db/lock/lockfakes"
"github.com/concourse/concourse/atc/exec"
@ -24,6 +23,7 @@ import (
"github.com/concourse/concourse/atc/exec/execfakes"
"github.com/concourse/concourse/atc/worker"
"github.com/concourse/concourse/atc/worker/workerfakes"
"github.com/concourse/concourse/vars"
)
var _ = Describe("TaskStep", func() {
@ -40,9 +40,8 @@ var _ = Describe("TaskStep", func() {
fakeLockFactory *lockfakes.FakeLockFactory
fakeSecretManager *credsfakes.FakeSecrets
fakeDelegate *execfakes.FakeTaskDelegate
taskPlan *atc.TaskPlan
fakeDelegate *execfakes.FakeTaskDelegate
taskPlan *atc.TaskPlan
interpolatedResourceTypes atc.VersionedResourceTypes
@ -52,6 +51,8 @@ var _ = Describe("TaskStep", func() {
taskStep exec.Step
stepErr error
credVarsTracker vars.CredVarsTracker
containerMetadata = db.ContainerMetadata{
WorkingDirectory: "some-artifact-root",
Type: db.ContainerTypeTask,
@ -79,10 +80,11 @@ var _ = Describe("TaskStep", func() {
fakeLockFactory = new(lockfakes.FakeLockFactory)
fakeSecretManager = new(credsfakes.FakeSecrets)
fakeSecretManager.GetReturns("super-secret-source", nil, true, nil)
credVars := vars.StaticVariables{"source-param": "super-secret-source"}
credVarsTracker = vars.NewCredVarsTracker(credVars, true)
fakeDelegate = new(execfakes.FakeTaskDelegate)
fakeDelegate.VariablesReturns(credVarsTracker)
fakeDelegate.StdoutReturns(stdoutBuf)
fakeDelegate.StderrReturns(stderrBuf)
@ -135,7 +137,6 @@ var _ = Describe("TaskStep", func() {
atc.ContainerLimits{},
stepMetadata,
containerMetadata,
fakeSecretManager,
fakeStrategy,
fakeClient,
fakeDelegate,
@ -222,6 +223,12 @@ var _ = Describe("TaskStep", func() {
})
It("secrets are tracked", func() {
mapit := vars.NewMapCredVarsTrackerIterator()
credVarsTracker.IterateInterpolatedCreds(mapit)
Expect(mapit.Data["source-param"]).To(Equal("super-secret-source"))
})
It("creates a containerSpec with the correct parameters", func() {
Expect(fakeClient.RunTaskStepCallCount()).To(Equal(1))

116
vars/vars_tracker.go Normal file
View File

@ -0,0 +1,116 @@
package vars
import (
"fmt"
"sync"
)
// CredVarsTracker implements the interface Variables. It wraps a secret manager and
// tracks key-values fetched from the secret managers. It also provides a method to
// thread-safely iterate interpolated key-values.
type CredVarsTrackerIterator interface {
YieldCred(string, string)
}
type CredVarsTracker interface {
Variables
IterateInterpolatedCreds(iter CredVarsTrackerIterator)
}
func NewCredVarsTracker(credVars Variables, on bool) CredVarsTracker {
if on {
return credVarsTracker{
credVars: credVars,
interpolatedCreds: map[string]string{},
lock: sync.RWMutex{},
}
} else {
return dummyCredVarsTracker{credVars: credVars}
}
}
type credVarsTracker struct {
credVars Variables
interpolatedCreds map[string]string
// Considering in-parallel steps, a lock is need.
lock sync.RWMutex
}
func (t credVarsTracker) Get(varDef VariableDefinition) (interface{}, bool, error) {
val, found, err := t.credVars.Get(varDef)
if found {
t.lock.Lock()
t.track(varDef.Name, val)
t.lock.Unlock()
}
return val, found, err
}
func (t credVarsTracker) track(name string, val interface{}) {
switch v := val.(type) {
case map[interface{}]interface{}:
for kk, vv := range v {
nn := fmt.Sprintf("%s.%s", name, kk.(string))
t.track(nn, vv)
}
case map[string]interface{}:
for kk, vv := range v {
nn := fmt.Sprintf("%s.%s", name, kk)
t.track(nn, vv)
}
case string:
t.interpolatedCreds[name] = v
default:
// Do nothing
}
}
func (t credVarsTracker) List() ([]VariableDefinition, error) {
return t.credVars.List()
}
func (t credVarsTracker) IterateInterpolatedCreds(iter CredVarsTrackerIterator) {
t.lock.RLock()
for k, v := range t.interpolatedCreds {
iter.YieldCred(k, v)
}
t.lock.RUnlock()
}
// DummyCredVarsTracker do nothing,
type dummyCredVarsTracker struct {
credVars Variables
}
func (t dummyCredVarsTracker) Get(varDef VariableDefinition) (interface{}, bool, error) {
return t.credVars.Get(varDef)
}
func (t dummyCredVarsTracker) List() ([]VariableDefinition, error) {
return t.credVars.List()
}
func (t dummyCredVarsTracker) IterateInterpolatedCreds(iter CredVarsTrackerIterator) {
// do nothing
}
// MapCredVarsTrackerIterator implements a simple CredVarsTrackerIterator which just
// populate interpolated secrets into a map. This could be useful in unit test.
type MapCredVarsTrackerIterator struct {
Data map[string]interface{}
}
func NewMapCredVarsTrackerIterator() (*MapCredVarsTrackerIterator) {
return &MapCredVarsTrackerIterator{
Data: map[string]interface{}{},
}
}
func (it *MapCredVarsTrackerIterator) YieldCred(k, v string) {
it.Data[k] = v
}

90
vars/vars_tracker_test.go Normal file
View File

@ -0,0 +1,90 @@
package vars_test
import (
. "github.com/concourse/concourse/vars"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("vars_tracker", func() {
var tracker CredVarsTracker
Describe("turn on track", func() {
BeforeEach(func() {
v := StaticVariables{"k1": "v1", "k2": "v2", "k3": "v3"}
tracker = NewCredVarsTracker(v, true)
})
Describe("Get", func() {
It("returns expected value", func() {
var (
val interface{}
found bool
err error
)
val, found, err = tracker.Get(VariableDefinition{Name: "k1"})
Expect(found).To(BeTrue())
Expect(err).To(BeNil())
Expect(val).To(Equal("v1"))
})
It("fetched variables are tracked", func() {
tracker.Get(VariableDefinition{Name: "k1"})
tracker.Get(VariableDefinition{Name: "k2"})
mapit := NewMapCredVarsTrackerIterator()
tracker.IterateInterpolatedCreds(mapit)
Expect(mapit.Data["k1"]).To(Equal("v1"))
Expect(mapit.Data["k2"]).To(Equal("v2"))
// "k3" has not been Get, thus should not be tracked.
Expect(mapit.Data["k3"]).To(BeNil())
})
})
Describe("List", func() {
It("returns list of names from multiple vars with duplicates", func() {
defs, err := tracker.List()
Expect(defs).To(ConsistOf([]VariableDefinition{{Name: "k1"}, {Name: "k2"}, {Name: "k3"}}))
Expect(err).ToNot(HaveOccurred())
})
})
})
Describe("turn off track", func() {
BeforeEach(func() {
v := StaticVariables{"k1": "v1", "k2": "v2", "k3": "v3"}
tracker = NewCredVarsTracker(v, false)
})
Describe("Get", func() {
It("returns expected value", func() {
var (
val interface{}
found bool
err error
)
val, found, err = tracker.Get(VariableDefinition{Name: "k1"})
Expect(found).To(BeTrue())
Expect(err).To(BeNil())
Expect(val).To(Equal("v1"))
})
It("fetched variables should not be tracked", func() {
tracker.Get(VariableDefinition{Name: "k1"})
tracker.Get(VariableDefinition{Name: "k2"})
mapit := NewMapCredVarsTrackerIterator()
tracker.IterateInterpolatedCreds(mapit)
Expect(mapit.Data["k1"]).To(BeNil())
Expect(mapit.Data["k2"]).To(BeNil())
Expect(mapit.Data["k3"]).To(BeNil())
})
})
Describe("List", func() {
It("returns list of names from multiple vars with duplicates", func() {
defs, err := tracker.List()
Expect(defs).To(ConsistOf([]VariableDefinition{{Name: "k1"}, {Name: "k2"}, {Name: "k3"}}))
Expect(err).ToNot(HaveOccurred())
})
})
})
})