deploy: Add include and exclude support for remote

This commit is contained in:
David Jones 2020-03-08 16:35:32 +00:00 committed by GitHub
parent cb12f41a96
commit 51e178a6a2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 98 additions and 4 deletions

View File

@ -138,7 +138,7 @@ func (d *Deployer) Deploy(ctx context.Context) error {
d.summary.NumLocal = len(local)
// Load remote files from the target.
remote, err := walkRemote(ctx, bucket)
remote, err := walkRemote(ctx, bucket, include, exclude)
if err != nil {
return err
}
@ -499,7 +499,7 @@ func walkLocal(fs afero.Fs, matchers []*matcher, include, exclude glob.Glob) (ma
}
// walkRemote walks the target bucket and returns a flat list.
func walkRemote(ctx context.Context, bucket *blob.Bucket) (map[string]*blob.ListObject, error) {
func walkRemote(ctx context.Context, bucket *blob.Bucket, include, exclude glob.Glob) (map[string]*blob.ListObject, error) {
retval := map[string]*blob.ListObject{}
iter := bucket.List(nil)
for {
@ -510,6 +510,15 @@ func walkRemote(ctx context.Context, bucket *blob.Bucket) (map[string]*blob.List
if err != nil {
return nil, err
}
// Check include/exclude matchers.
if include != nil && !include.Match(obj.Key) {
jww.INFO.Printf(" remote dropping %q due to include\n", obj.Key)
continue
}
if exclude != nil && exclude.Match(obj.Key) {
jww.INFO.Printf(" remote dropping %q due to exclude\n", obj.Key)
continue
}
// If the remote didn't give us an MD5, compute one.
// This can happen for some providers (e.g., fileblob, which uses the
// local filesystem), but not for the most common Cloud providers

View File

@ -720,6 +720,88 @@ func TestIncludeExclude(t *testing.T) {
}
}
// TestIncludeExcludeRemoteDelete verifies deleted local files that don't match include/exclude patterns
// are not deleted on the remote.
func TestIncludeExcludeRemoteDelete(t *testing.T) {
ctx := context.Background()
tests := []struct {
Include string
Exclude string
Want deploySummary
}{
{
Want: deploySummary{NumLocal: 3, NumRemote: 5, NumUploads: 0, NumDeletes: 2},
},
{
Include: "**aaa",
Want: deploySummary{NumLocal: 2, NumRemote: 3, NumUploads: 0, NumDeletes: 1},
},
{
Include: "subdir/**",
Want: deploySummary{NumLocal: 1, NumRemote: 2, NumUploads: 0, NumDeletes: 1},
},
{
Exclude: "**bbb",
Want: deploySummary{NumLocal: 2, NumRemote: 3, NumUploads: 0, NumDeletes: 1},
},
{
Exclude: "bbb",
Want: deploySummary{NumLocal: 3, NumRemote: 4, NumUploads: 0, NumDeletes: 1},
},
}
for _, test := range tests {
t.Run(fmt.Sprintf("include %q exclude %q", test.Include, test.Exclude), func(t *testing.T) {
fsTests, cleanup, err := initFsTests()
if err != nil {
t.Fatal(err)
}
defer cleanup()
fsTest := fsTests[1] // just do file-based test
local, err := initLocalFs(ctx, fsTest.fs)
if err != nil {
t.Fatal(err)
}
deployer := &Deployer{
localFs: fsTest.fs,
maxDeletes: -1,
bucket: fsTest.bucket,
}
// Initial sync to get the files on the remote
if err := deployer.Deploy(ctx); err != nil {
t.Errorf("deploy: failed: %v", err)
}
// Delete two files, [1] and [2].
if err := fsTest.fs.Remove(local[1].Name); err != nil {
t.Fatal(err)
}
if err := fsTest.fs.Remove(local[2].Name); err != nil {
t.Fatal(err)
}
// Second sync
tgt := &target{
Include: test.Include,
Exclude: test.Exclude,
}
if err := tgt.parseIncludeExclude(); err != nil {
t.Error(err)
}
deployer.target = tgt
if err := deployer.Deploy(ctx); err != nil {
t.Errorf("deploy: failed: %v", err)
}
if !cmp.Equal(deployer.summary, test.Want) {
t.Errorf("deploy: got %v, want %v", deployer.summary, test.Want)
}
})
}
}
// TestCompression verifies that gzip compression works correctly.
// In particular, MD5 hashes must be of the compressed content.
func TestCompression(t *testing.T) {

View File

@ -85,8 +85,11 @@ cloudFrontDistributionID = <ID>
# Optionally, you can include or exclude specific files.
# See https://godoc.org/github.com/gobwas/glob#Glob for the glob pattern syntax.
# If non-empty, the pattern is matched against the local path.
# If exclude is non-empty, and a file's path matches it, that file is dropped.
# If include is non-empty, and a file's path does not match it, that file is dropped.
# All paths are matched against in their filepath.ToSlash form.
# If exclude is non-empty, and a local or remote file's path matches it, that file is not synced.
# If include is non-empty, and a local or remote file's path does not match it, that file is not synced.
# As a result, local files that don't pass the include/exclude filters are not uploaded to remote,
# and remote files that don't pass the include/exclude filters are not deleted.
# include = "**.html" # would only include files with ".html" suffix
# exclude = "**.{jpg, png}" # would exclude files with ".jpg" or ".png" suffix