From dfc214115b47473272e3709fb978ef47e05dc76d Mon Sep 17 00:00:00 2001 From: Stoica-Marcu Floris-Andrei Date: Tue, 22 Sep 2020 12:16:05 +0300 Subject: [PATCH 1/3] Add stack config command Make use of existing modules and functions in order to output the merged configs. Added skip interpolation flag of variables, so that you can pipe the output back to stack deploy without much hassle. Signed-off-by: Stoica-Marcu Floris-Andrei Signed-off-by: Sebastiaan van Stijn --- cli/command/stack/cmd.go | 1 + cli/command/stack/config.go | 60 ++++++++++++ cli/command/stack/config_test.go | 106 +++++++++++++++++++++ cli/command/stack/loader/loader.go | 5 +- cli/command/stack/loader/loader_test.go | 4 +- cli/command/stack/options/opts.go | 6 ++ docs/reference/commandline/index.md | 15 +-- docs/reference/commandline/stack_config.md | 68 +++++++++++++ docs/reference/commandline/stack_deploy.md | 1 + e2e/stack/config_test.go | 12 +++ 10 files changed, 267 insertions(+), 11 deletions(-) create mode 100644 cli/command/stack/config.go create mode 100644 cli/command/stack/config_test.go create mode 100644 docs/reference/commandline/stack_config.md create mode 100644 e2e/stack/config_test.go diff --git a/cli/command/stack/cmd.go b/cli/command/stack/cmd.go index 4f59b99f4a..e2027568bc 100644 --- a/cli/command/stack/cmd.go +++ b/cli/command/stack/cmd.go @@ -33,6 +33,7 @@ func NewStackCommand(dockerCli command.Cli) *cobra.Command { newPsCommand(dockerCli), newRemoveCommand(dockerCli), newServicesCommand(dockerCli), + newConfigCommand(dockerCli), ) flags := cmd.PersistentFlags() flags.String("orchestrator", "", "Orchestrator to use (swarm|all)") diff --git a/cli/command/stack/config.go b/cli/command/stack/config.go new file mode 100644 index 0000000000..c6bf74f1e3 --- /dev/null +++ b/cli/command/stack/config.go @@ -0,0 +1,60 @@ +package stack + +import ( + "fmt" + + "github.com/docker/cli/cli" + "github.com/docker/cli/cli/command" + "github.com/docker/cli/cli/command/stack/loader" + "github.com/docker/cli/cli/command/stack/options" + composeLoader "github.com/docker/cli/cli/compose/loader" + composetypes "github.com/docker/cli/cli/compose/types" + "github.com/spf13/cobra" + yaml "gopkg.in/yaml.v2" +) + +func newConfigCommand(dockerCli command.Cli) *cobra.Command { + var opts options.Config + + cmd := &cobra.Command{ + Use: "config [OPTIONS]", + Short: "Outputs the final config file, after doing merges and interpolations", + Args: cli.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + configDetails, err := loader.GetConfigDetails(opts.Composefiles, dockerCli.In()) + if err != nil { + return err + } + + cfg, err := outputConfig(configDetails, opts.SkipInterpolation) + if err != nil { + return err + } + + _, err = fmt.Fprintf(dockerCli.Out(), "%s", cfg) + return err + }, + } + + flags := cmd.Flags() + flags.StringSliceVarP(&opts.Composefiles, "compose-file", "c", []string{}, `Path to a Compose file, or "-" to read from stdin`) + flags.BoolVar(&opts.SkipInterpolation, "skip-interpolation", false, "Skip interpolation and output only merged config") + return cmd +} + +// outputConfig returns the merged and interpolated config file +func outputConfig(configFiles composetypes.ConfigDetails, skipInterpolation bool) (string, error) { + optsFunc := func(options *composeLoader.Options) { + options.SkipInterpolation = skipInterpolation + } + config, err := composeLoader.Load(configFiles, optsFunc) + if err != nil { + return "", err + } + + d, err := yaml.Marshal(&config) + if err != nil { + return "", err + } + return string(d), nil +} diff --git a/cli/command/stack/config_test.go b/cli/command/stack/config_test.go new file mode 100644 index 0000000000..80b121f5a9 --- /dev/null +++ b/cli/command/stack/config_test.go @@ -0,0 +1,106 @@ +package stack + +import ( + "io" + "testing" + + "github.com/docker/cli/cli/compose/loader" + composetypes "github.com/docker/cli/cli/compose/types" + "github.com/docker/cli/internal/test" + "gotest.tools/v3/assert" +) + +func TestConfigWithEmptyComposeFile(t *testing.T) { + cmd := newConfigCommand(test.NewFakeCli(&fakeClient{})) + cmd.SetOut(io.Discard) + + assert.ErrorContains(t, cmd.Execute(), `Please specify a Compose file`) +} + +var configMergeTests = []struct { + name string + skipInterpolation bool + first string + second string + merged string +}{ + { + name: "With Interpolation", + skipInterpolation: false, + first: `version: "3.7" +services: + foo: + image: busybox:latest + command: cat file1.txt +`, + second: `version: "3.7" +services: + foo: + image: busybox:${VERSION} + command: cat file2.txt +`, + merged: `version: "3.7" +services: + foo: + command: + - cat + - file2.txt + image: busybox:1.0 +`, + }, + { + name: "Without Interpolation", + skipInterpolation: true, + first: `version: "3.7" +services: + foo: + image: busybox:latest + command: cat file1.txt +`, + second: `version: "3.7" +services: + foo: + image: busybox:${VERSION} + command: cat file2.txt +`, + merged: `version: "3.7" +services: + foo: + command: + - cat + - file2.txt + image: busybox:${VERSION} +`, + }, +} + +func TestConfigMergeInterpolation(t *testing.T) { + + for _, tt := range configMergeTests { + t.Run(tt.name, func(t *testing.T) { + firstConfig := []byte(tt.first) + secondConfig := []byte(tt.second) + + firstConfigData, err := loader.ParseYAML(firstConfig) + assert.NilError(t, err) + secondConfigData, err := loader.ParseYAML(secondConfig) + assert.NilError(t, err) + + env := map[string]string{ + "VERSION": "1.0", + } + + cfg, err := outputConfig(composetypes.ConfigDetails{ + ConfigFiles: []composetypes.ConfigFile{ + {Config: firstConfigData, Filename: "firstConfig"}, + {Config: secondConfigData, Filename: "secondConfig"}, + }, + Environment: env, + }, tt.skipInterpolation) + assert.NilError(t, err) + + assert.Equal(t, cfg, tt.merged) + }) + } + +} diff --git a/cli/command/stack/loader/loader.go b/cli/command/stack/loader/loader.go index a61efc7bd7..22352f434e 100644 --- a/cli/command/stack/loader/loader.go +++ b/cli/command/stack/loader/loader.go @@ -18,7 +18,7 @@ import ( // LoadComposefile parse the composefile specified in the cli and returns its Config and version. func LoadComposefile(dockerCli command.Cli, opts options.Deploy) (*composetypes.Config, error) { - configDetails, err := getConfigDetails(opts.Composefiles, dockerCli.In()) + configDetails, err := GetConfigDetails(opts.Composefiles, dockerCli.In()) if err != nil { return nil, err } @@ -68,7 +68,8 @@ func propertyWarnings(properties map[string]string) string { return strings.Join(msgs, "\n\n") } -func getConfigDetails(composefiles []string, stdin io.Reader) (composetypes.ConfigDetails, error) { +// GetConfigDetails parse the composefiles specified in the cli and returns their ConfigDetails +func GetConfigDetails(composefiles []string, stdin io.Reader) (composetypes.ConfigDetails, error) { var details composetypes.ConfigDetails if len(composefiles) == 0 { diff --git a/cli/command/stack/loader/loader_test.go b/cli/command/stack/loader/loader_test.go index fd504bc9d1..6ddca65bb8 100644 --- a/cli/command/stack/loader/loader_test.go +++ b/cli/command/stack/loader/loader_test.go @@ -21,7 +21,7 @@ services: file := fs.NewFile(t, "test-get-config-details", fs.WithContent(content)) defer file.Remove() - details, err := getConfigDetails([]string{file.Path()}, nil) + details, err := GetConfigDetails([]string{file.Path()}, nil) assert.NilError(t, err) assert.Check(t, is.Equal(filepath.Dir(file.Path()), details.WorkingDir)) assert.Assert(t, is.Len(details.ConfigFiles, 1)) @@ -36,7 +36,7 @@ services: foo: image: alpine:3.5 ` - details, err := getConfigDetails([]string{"-"}, strings.NewReader(content)) + details, err := GetConfigDetails([]string{"-"}, strings.NewReader(content)) assert.NilError(t, err) cwd, err := os.Getwd() assert.NilError(t, err) diff --git a/cli/command/stack/options/opts.go b/cli/command/stack/options/opts.go index 9842b4995c..9c0cece153 100644 --- a/cli/command/stack/options/opts.go +++ b/cli/command/stack/options/opts.go @@ -11,6 +11,12 @@ type Deploy struct { Prune bool } +// Config holds docker stack config options +type Config struct { + Composefiles []string + SkipInterpolation bool +} + // List holds docker stack ls options type List struct { Format string diff --git a/docs/reference/commandline/index.md b/docs/reference/commandline/index.md index 9cd7ea2ad7..0dc074d676 100644 --- a/docs/reference/commandline/index.md +++ b/docs/reference/commandline/index.md @@ -152,13 +152,14 @@ read the [`dockerd`](dockerd.md) reference page. ### Swarm stack commands -| Command | Description | -|:------------------------------------|:-----------------------------------------------| -| [stack deploy](stack_deploy.md) | Deploy a new stack or update an existing stack | -| [stack ls](stack_ls.md) | List stacks in the swarm | -| [stack ps](stack_ps.md) | List the tasks in the stack | -| [stack rm](stack_rm.md) | Remove the stack from the swarm | -| [stack services](stack_services.md) | List the services in the stack | +| Command | Description | +|:------------------------------------|:--------------------------------------------------------| +| [stack deploy](stack_deploy.md) | Deploy a new stack or update an existing stack | +| [stack ls](stack_ls.md) | List stacks in the swarm | +| [stack ps](stack_ps.md) | List the tasks in the stack | +| [stack rm](stack_rm.md) | Remove the stack from the swarm | +| [stack services](stack_services.md) | List the services in the stack | +| [stack config](stack_config.md) | Output the Compose file after merges and interpolations | ### Plugin commands diff --git a/docs/reference/commandline/stack_config.md b/docs/reference/commandline/stack_config.md new file mode 100644 index 0000000000..0d3ff490ea --- /dev/null +++ b/docs/reference/commandline/stack_config.md @@ -0,0 +1,68 @@ +--- +title: "stack config" +description: "The stack config command description and usage" +keywords: "stack, config" +--- + +# stack config + +```markdown +Usage: docker stack config [OPTIONS] + +Outputs the final config file, after doing merges and interpolations + +Aliases: + config, cfg + +Options: + -c, --compose-file strings Path to a Compose file, or "-" to read from stdin + --orchestrator string Orchestrator to use (swarm|kubernetes|all) + --skip-interpolation Skip interpolation and output only merged config +``` + +## Description + +Outputs the final Compose file, after doing the merges and interpolations of the input Compose files. + +## Examples + +The following command outputs the result of the merge and interpolation of two Compose files. + +```bash +$ docker stack config --compose-file docker-compose.yml --compose-file docker-compose.prod.yml +``` + +The Compose file can also be provided as standard input with `--compose-file -`: + +```bash +$ cat docker-compose.yml | docker stack config --compose-file - +``` + +### Skipping interpolation + +In some cases, it might be useful to skip interpolation of environment variables. +For example, when you want to pipe the output of this command back to `stack deploy`. + +If you have a regex for a redirect route in an environment variable for your webserver you would use two `$` signs to prevent `stack deploy` from interpolating `${1}`. + +```bash + service: webserver + environment: + REDIRECT_REGEX=http://host/redirect/$${1} +``` + +With interpolation, the `stack config` command will replace the environment variable in the Compose file +with `REDIRECT_REGEX=http://host/redirect/${1}`, but then when piping it back to the `stack deploy` +command it will be interpolated again and result in undefined behavior. +That is why, when piping the output back to `stack deploy` one should always prefer the `--skip-interpolation` option. + +``` +$ docker stack config --compose-file web.yml --compose-file web.prod.yml --skip-interpolation | docker stack deploy --compose-file - +``` + +## Related commands + +* [stack deploy](stack_deploy.md) +* [stack ps](stack_ps.md) +* [stack rm](stack_rm.md) +* [stack services](stack_services.md) diff --git a/docs/reference/commandline/stack_deploy.md b/docs/reference/commandline/stack_deploy.md index fcce61643f..f1989aecd7 100644 --- a/docs/reference/commandline/stack_deploy.md +++ b/docs/reference/commandline/stack_deploy.md @@ -111,3 +111,4 @@ axqh55ipl40h vossibility_vossibility-collector replicated 1/1 icecrime/ * [stack ps](stack_ps.md) * [stack rm](stack_rm.md) * [stack services](stack_services.md) +* [stack config](stack_config.md) diff --git a/e2e/stack/config_test.go b/e2e/stack/config_test.go new file mode 100644 index 0000000000..1d56ae7019 --- /dev/null +++ b/e2e/stack/config_test.go @@ -0,0 +1,12 @@ +package stack + +import ( + "testing" + + "gotest.tools/v3/icmd" +) + +func TestConfigFullStack(t *testing.T) { + result := icmd.RunCommand("docker", "stack", "config", "--compose-file=./testdata/full-stack.yml") + result.Assert(t, icmd.Success) +} From fbea85d4721a528fad9f54cacfd526df2dce2508 Mon Sep 17 00:00:00 2001 From: Stoica-Marcu Floris-Andrei Date: Thu, 24 Sep 2020 04:37:13 +0300 Subject: [PATCH 2/3] Change merge strategy for service volumes Signed-off-by: Stoica-Marcu Floris-Andrei --- cli/compose/loader/merge.go | 31 +++++++++++++ cli/compose/loader/merge_test.go | 75 ++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/cli/compose/loader/merge.go b/cli/compose/loader/merge.go index 0de8d8a1b6..a2bdabec73 100644 --- a/cli/compose/loader/merge.go +++ b/cli/compose/loader/merge.go @@ -58,6 +58,7 @@ func mergeServices(base, override []types.ServiceConfig) ([]types.ServiceConfig, reflect.TypeOf([]types.ServiceSecretConfig{}): mergeSlice(toServiceSecretConfigsMap, toServiceSecretConfigsSlice), reflect.TypeOf([]types.ServiceConfigObjConfig{}): mergeSlice(toServiceConfigObjConfigsMap, toSServiceConfigObjConfigsSlice), reflect.TypeOf(&types.UlimitsConfig{}): mergeUlimitsConfig, + reflect.TypeOf([]types.ServiceVolumeConfig{}): mergeSlice(toServiceVolumeConfigsMap, toServiceVolumeConfigsSlice), reflect.TypeOf(&types.ServiceNetworkConfig{}): mergeServiceNetworkConfig, }, } @@ -116,6 +117,18 @@ func toServicePortConfigsMap(s interface{}) (map[interface{}]interface{}, error) return m, nil } +func toServiceVolumeConfigsMap(s interface{}) (map[interface{}]interface{}, error) { + volumes, ok := s.([]types.ServiceVolumeConfig) + if !ok { + return nil, errors.Errorf("not a serviceVolumeConfig slice: %v", s) + } + m := map[interface{}]interface{}{} + for _, v := range volumes { + m[v.Target] = v + } + return m, nil +} + func toServiceSecretConfigsSlice(dst reflect.Value, m map[interface{}]interface{}) error { s := []types.ServiceSecretConfig{} for _, v := range m { @@ -146,6 +159,16 @@ func toServicePortConfigsSlice(dst reflect.Value, m map[interface{}]interface{}) return nil } +func toServiceVolumeConfigsSlice(dst reflect.Value, m map[interface{}]interface{}) error { + s := []types.ServiceVolumeConfig{} + for _, v := range m { + s = append(s, v.(types.ServiceVolumeConfig)) + } + sort.Slice(s, func(i, j int) bool { return s[i].Target < s[j].Target }) + dst.Set(reflect.ValueOf(s)) + return nil +} + type tomapFn func(s interface{}) (map[interface{}]interface{}, error) type writeValueFromMapFn func(reflect.Value, map[interface{}]interface{}) error @@ -211,6 +234,14 @@ func mergeUlimitsConfig(dst, src reflect.Value) error { return nil } +//nolint: unparam +func mergeServiceVolumeConfig(dst, src reflect.Value) error { + if dst.Elem().FieldByName("target").String() == src.Elem().FieldByName("target").String() { + dst.Set(src.Elem()) + } + return nil +} + //nolint: unparam func mergeServiceNetworkConfig(dst, src reflect.Value) error { if src.Interface() != reflect.Zero(reflect.TypeOf(src.Interface())).Interface() { diff --git a/cli/compose/loader/merge_test.go b/cli/compose/loader/merge_test.go index 4828df8548..c5bbf3b5de 100644 --- a/cli/compose/loader/merge_test.go +++ b/cli/compose/loader/merge_test.go @@ -1017,6 +1017,81 @@ func TestLoadMultipleNetworks(t *testing.T) { }, config) } +func TestLoadMultipleServiceVolumes(t *testing.T) { + base := map[string]interface{}{ + "version": "3.7", + "services": map[string]interface{}{ + "foo": map[string]interface{}{ + "image": "baz", + "volumes": []interface{}{ + map[string]interface{}{ + "type": "volume", + "source": "sourceVolume", + "target": "/var/app", + }, + }, + }, + }, + "volumes": map[string]interface{}{ + "sourceVolume": map[string]interface{}{}, + }, + "networks": map[string]interface{}{}, + "secrets": map[string]interface{}{}, + "configs": map[string]interface{}{}, + } + override := map[string]interface{}{ + "version": "3.7", + "services": map[string]interface{}{ + "foo": map[string]interface{}{ + "image": "baz", + "volumes": []interface{}{ + map[string]interface{}{ + "type": "volume", + "source": "/local", + "target": "/var/app", + }, + }, + }, + }, + "volumes": map[string]interface{}{}, + "networks": map[string]interface{}{}, + "secrets": map[string]interface{}{}, + "configs": map[string]interface{}{}, + } + configDetails := types.ConfigDetails{ + ConfigFiles: []types.ConfigFile{ + {Filename: "base.yml", Config: base}, + {Filename: "override.yml", Config: override}, + }, + } + config, err := Load(configDetails) + assert.NilError(t, err) + assert.DeepEqual(t, &types.Config{ + Filename: "base.yml", + Version: "3.7", + Services: []types.ServiceConfig{ + { + Name: "foo", + Image: "baz", + Environment: types.MappingWithEquals{}, + Volumes: []types.ServiceVolumeConfig{ + { + Type: "volume", + Source: "/local", + Target: "/var/app", + }, + }, + }, + }, + Volumes: map[string]types.VolumeConfig{ + "sourceVolume": {}, + }, + Secrets: map[string]types.SecretConfig{}, + Configs: map[string]types.ConfigObjConfig{}, + Networks: map[string]types.NetworkConfig{}, + }, config) +} + func TestMergeUlimitsConfig(t *testing.T) { specials := &specials{ m: map[reflect.Type]func(dst, src reflect.Value) error{ From cff702d889d4487a39234fa51c5ca49e3d57d052 Mon Sep 17 00:00:00 2001 From: Stoica-Marcu Floris-Andrei Date: Wed, 23 Sep 2020 03:33:30 +0300 Subject: [PATCH 3/3] Add merge to ShellCommand properties in config Signed-off-by: Stoica-Marcu Floris-Andrei --- cli/compose/loader/merge.go | 7 +++-- cli/compose/loader/merge_test.go | 53 ++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/cli/compose/loader/merge.go b/cli/compose/loader/merge.go index a2bdabec73..97d55ea6b8 100644 --- a/cli/compose/loader/merge.go +++ b/cli/compose/loader/merge.go @@ -59,6 +59,7 @@ func mergeServices(base, override []types.ServiceConfig) ([]types.ServiceConfig, reflect.TypeOf([]types.ServiceConfigObjConfig{}): mergeSlice(toServiceConfigObjConfigsMap, toSServiceConfigObjConfigsSlice), reflect.TypeOf(&types.UlimitsConfig{}): mergeUlimitsConfig, reflect.TypeOf([]types.ServiceVolumeConfig{}): mergeSlice(toServiceVolumeConfigsMap, toServiceVolumeConfigsSlice), + reflect.TypeOf(types.ShellCommand{}): mergeShellCommand, reflect.TypeOf(&types.ServiceNetworkConfig{}): mergeServiceNetworkConfig, }, } @@ -235,9 +236,9 @@ func mergeUlimitsConfig(dst, src reflect.Value) error { } //nolint: unparam -func mergeServiceVolumeConfig(dst, src reflect.Value) error { - if dst.Elem().FieldByName("target").String() == src.Elem().FieldByName("target").String() { - dst.Set(src.Elem()) +func mergeShellCommand(dst, src reflect.Value) error { + if src.Len() != 0 { + dst.Set(src) } return nil } diff --git a/cli/compose/loader/merge_test.go b/cli/compose/loader/merge_test.go index c5bbf3b5de..6f2262b072 100644 --- a/cli/compose/loader/merge_test.go +++ b/cli/compose/loader/merge_test.go @@ -1017,6 +1017,59 @@ func TestLoadMultipleNetworks(t *testing.T) { }, config) } +func TestLoadMultipleServiceCommands(t *testing.T) { + base := map[string]interface{}{ + "version": "3.7", + "services": map[string]interface{}{ + "foo": map[string]interface{}{ + "image": "baz", + "command": "foo bar", + }, + }, + "volumes": map[string]interface{}{}, + "networks": map[string]interface{}{}, + "secrets": map[string]interface{}{}, + "configs": map[string]interface{}{}, + } + override := map[string]interface{}{ + "version": "3.7", + "services": map[string]interface{}{ + "foo": map[string]interface{}{ + "image": "baz", + "command": "foo baz", + }, + }, + "volumes": map[string]interface{}{}, + "networks": map[string]interface{}{}, + "secrets": map[string]interface{}{}, + "configs": map[string]interface{}{}, + } + configDetails := types.ConfigDetails{ + ConfigFiles: []types.ConfigFile{ + {Filename: "base.yml", Config: base}, + {Filename: "override.yml", Config: override}, + }, + } + config, err := Load(configDetails) + assert.NilError(t, err) + assert.DeepEqual(t, &types.Config{ + Filename: "base.yml", + Version: "3.7", + Services: []types.ServiceConfig{ + { + Name: "foo", + Image: "baz", + Command: types.ShellCommand{"foo", "baz"}, + Environment: types.MappingWithEquals{}, + }, + }, + Volumes: map[string]types.VolumeConfig{}, + Secrets: map[string]types.SecretConfig{}, + Configs: map[string]types.ConfigObjConfig{}, + Networks: map[string]types.NetworkConfig{}, + }, config) +} + func TestLoadMultipleServiceVolumes(t *testing.T) { base := map[string]interface{}{ "version": "3.7",