diff --git a/cmd/admin-bucket-remote.go b/cmd/admin-bucket-remote.go index c69e38b7..31ea1e28 100644 --- a/cmd/admin-bucket-remote.go +++ b/cmd/admin-bucket-remote.go @@ -20,7 +20,7 @@ import "github.com/minio/cli" var adminBucketRemoteCmd = cli.Command{ Name: "remote", - Usage: "Manage bucket targets", + Usage: "manage remote bucket targets", Action: mainadminBucketRemote, Before: setGlobalsFromContext, Flags: globalFlags, diff --git a/cmd/auto-complete.go b/cmd/auto-complete.go index c4f9ffe0..874865ab 100644 --- a/cmd/auto-complete.go +++ b/cmd/auto-complete.go @@ -255,6 +255,13 @@ var completeCmds = map[string]complete.Predictor{ "/encrypt/info": s3Complete{deepLevel: 2}, "/encrypt/clear": s3Complete{deepLevel: 2}, + "/replicate/add": s3Complete{deepLevel: 2}, + "/replicate/set": s3Complete{deepLevel: 2}, + "/replicate/ls": s3Complete{deepLevel: 2}, + "/replicate/rm": s3Complete{deepLevel: 2}, + "/replicate/export": s3Complete{deepLevel: 2}, + "/replicate/import": s3Complete{deepLevel: 2}, + "/tag/list": s3Completer, "/tag/remove": s3Completer, "/tag/set": s3Completer, @@ -324,11 +331,10 @@ var completeCmds = map[string]complete.Predictor{ "/admin/group/remove": aliasCompleter, "/admin/group/info": aliasCompleter, - "/admin/bucket/replication": aliasCompleter, - "/admin/bucket/remote": aliasCompleter, - "/config/host/add": nil, - "/config/host/list": aliasCompleter, - "/config/host/remove": aliasCompleter, + "/admin/bucket/remote": aliasCompleter, + "/config/host/add": nil, + "/config/host/list": aliasCompleter, + "/config/host/remove": aliasCompleter, "/update": nil, } diff --git a/cmd/client-fs.go b/cmd/client-fs.go index 79b84636..494de9ed 100644 --- a/cmd/client-fs.go +++ b/cmd/client-fs.go @@ -1155,7 +1155,7 @@ func (f *fsClient) GetReplication(ctx context.Context) (replication.Config, *pro } // Set replication configuration for a given bucket, not implemented. -func (f *fsClient) SetReplication(ctx context.Context, cfg *replication.Config) *probe.Error { +func (f *fsClient) SetReplication(ctx context.Context, cfg *replication.Config, opts replication.Options) *probe.Error { return probe.NewError(APINotImplemented{ API: "SetReplication", APIType: "filesystem", diff --git a/cmd/client-s3.go b/cmd/client-s3.go index 595b8269..d900ae60 100644 --- a/cmd/client-s3.go +++ b/cmd/client-s3.go @@ -2452,12 +2452,29 @@ func (c *S3Client) RemoveReplication(ctx context.Context) *probe.Error { } // SetReplication sets replication configuration for a given bucket. -func (c *S3Client) SetReplication(ctx context.Context, cfg *replication.Config) *probe.Error { - bucket, _ := c.url2BucketAndObject() +func (c *S3Client) SetReplication(ctx context.Context, cfg *replication.Config, opts replication.Options) *probe.Error { + bucket, objectPrefix := c.url2BucketAndObject() if bucket == "" { return probe.NewError(BucketNameEmpty{}) } - + opts.Prefix = objectPrefix + switch opts.Op { + case replication.AddOption: + if e := cfg.AddRule(opts); e != nil { + return probe.NewError(e) + } + case replication.SetOption: + if e := cfg.EditRule(opts); e != nil { + return probe.NewError(e) + } + case replication.RemoveOption: + if e := cfg.RemoveRule(opts); e != nil { + return probe.NewError(e) + } + case replication.ImportOption: + default: + return probe.NewError(fmt.Errorf("Invalid replication option")) + } if e := c.api.SetBucketReplication(ctx, bucket, *cfg); e != nil { return probe.NewError(e) } diff --git a/cmd/client.go b/cmd/client.go index df999206..e3f101e1 100644 --- a/cmd/client.go +++ b/cmd/client.go @@ -141,7 +141,7 @@ type Client interface { SetVersioning(ctx context.Context, status string) *probe.Error // Replication operations GetReplication(ctx context.Context) (replication.Config, *probe.Error) - SetReplication(ctx context.Context, cfg *replication.Config) *probe.Error + SetReplication(ctx context.Context, cfg *replication.Config, opts replication.Options) *probe.Error RemoveReplication(ctx context.Context) *probe.Error // Encryption operations GetEncryption(ctx context.Context) (string, string, *probe.Error) diff --git a/cmd/main.go b/cmd/main.go index a34dabc1..8256ac67 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -347,6 +347,7 @@ var appCmds = []cli.Command{ watchCmd, policyCmd, tagCmd, + replicateCmd, adminCmd, configCmd, updateCmd, diff --git a/cmd/replicate-add.go b/cmd/replicate-add.go new file mode 100644 index 00000000..ce91cd70 --- /dev/null +++ b/cmd/replicate-add.go @@ -0,0 +1,150 @@ +/* + * MinIO Client (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "context" + "strconv" + + "github.com/fatih/color" + "github.com/minio/cli" + json "github.com/minio/mc/pkg/colorjson" + "github.com/minio/mc/pkg/probe" + "github.com/minio/minio-go/v7/pkg/replication" + "github.com/minio/minio/pkg/console" +) + +var replicateAddFlags = []cli.Flag{ + cli.StringFlag{ + Name: "arn", + Usage: "Role Arn", + }, + cli.StringFlag{ + Name: "id", + Usage: "id for the rule, should be a unique value", + }, + cli.StringFlag{ + Name: "tags", + Usage: "format '=&=&=', multiple values allowed for multiple key/value pairs", + }, + cli.StringFlag{ + Name: "storage-class", + Usage: "storage class for destination (STANDARD_IA,REDUCED_REDUNDANCY etc)", + }, + cli.BoolFlag{ + Name: "disable", + Usage: "disable the rule", + }, + cli.IntFlag{ + Name: "priority", + Usage: "priority of the rule, should be unique and is a required field", + }, +} + +var replicateAddCmd = cli.Command{ + Name: "add", + Usage: "add a server side replication configuration rule", + Action: mainReplicateAdd, + Before: setGlobalsFromContext, + Flags: append(globalFlags, replicateAddFlags...), + CustomHelpTemplate: `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}} TARGET + +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}} +EXAMPLES: + 1. Add replication configuration rule on bucket "mybucket" for alias "myminio". + {{.Prompt}} {{.HelpName}} myminio/mybucket/prefix --tags "key1=value1&key2=value2" \ + --storage-class "STANDARD" \ + --arn 'arn:minio:replica::c5be6b16-769d-432a-9ef1-4567081f3566:destbucket' --priority 1 + + 2. Add replication configuration rule with Disabled status on bucket "mybucket" for alias "myminio". + {{.Prompt}} {{.HelpName}} myminio/mybucket/prefix --tags "key1=value1&key2=value2" \ + --storage-class "STANDARD" --disable + --arn 'arn:minio:replica::c5be6b16-769d-432a-9ef1-4567081f3566:destbucket' --priority 1 +`, +} + +// checkReplicateAddSyntax - validate all the passed arguments +func checkReplicateAddSyntax(ctx *cli.Context) { + if len(ctx.Args()) != 1 { + cli.ShowCommandHelpAndExit(ctx, "add", 1) // last argument is exit code + } +} + +type replicateAddMessage struct { + Op string `json:"op"` + Status string `json:"status"` + URL string `json:"url"` + ID string `json:"id"` +} + +func (l replicateAddMessage) JSON() string { + l.Status = "success" + jsonMessageBytes, e := json.MarshalIndent(l, "", " ") + fatalIf(probe.NewError(e), "Unable to marshal into JSON.") + return string(jsonMessageBytes) +} + +func (l replicateAddMessage) String() string { + if l.ID != "" { + return console.Colorize("replicateAddMessage", "Replication configuration rule with ID `"+l.ID+"` applied to "+l.URL+".") + } + return console.Colorize("replicateAddMessage", "Replication configuration rule applied to "+l.URL+" successfully.") +} + +func mainReplicateAdd(cliCtx *cli.Context) error { + ctx, cancelReplicateAdd := context.WithCancel(globalContext) + defer cancelReplicateAdd() + + console.SetColor("replicateAddMessage", color.New(color.FgGreen)) + + checkReplicateAddSyntax(cliCtx) + + // Get the alias parameter from cli + args := cliCtx.Args() + aliasedURL := args.Get(0) + // Create a new Client + client, err := newClient(aliasedURL) + fatalIf(err, "Unable to initialize connection.") + rcfg, err := client.GetReplication(ctx) + fatalIf(err.Trace(args...), "Unable to get replication configuration") + ruleStatus := "enable" + if cliCtx.Bool("disable") { + ruleStatus = "disable" + } + opts := replication.Options{ + TagString: cliCtx.String("tags"), + Arn: cliCtx.String("arn"), + StorageClass: cliCtx.String("storage-class"), + Priority: strconv.Itoa(cliCtx.Int("priority")), + RuleStatus: ruleStatus, + ID: cliCtx.String("id"), + Op: replication.AddOption, + } + fatalIf(client.SetReplication(ctx, &rcfg, opts), "Could not add replication rule") + printMsg(replicateAddMessage{ + Op: "add", + URL: aliasedURL, + ID: opts.ID, + }) + return nil +} diff --git a/cmd/replicate-export.go b/cmd/replicate-export.go new file mode 100644 index 00000000..a2c4e8b8 --- /dev/null +++ b/cmd/replicate-export.go @@ -0,0 +1,108 @@ +/* + * MinIO Client (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "context" + + "github.com/fatih/color" + "github.com/minio/cli" + json "github.com/minio/mc/pkg/colorjson" + "github.com/minio/mc/pkg/probe" + "github.com/minio/minio-go/v7/pkg/replication" + "github.com/minio/minio/pkg/console" +) + +var replicateExportCmd = cli.Command{ + Name: "export", + Usage: "export server side replication configuration", + Action: mainReplicateExport, + Before: setGlobalsFromContext, + Flags: globalFlags, + CustomHelpTemplate: `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}} TARGET + +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}} +EXAMPLES: + 1. Print replication configuration on bucket "mybucket" for alias "myminio" to STDOUT. + {{.Prompt}} {{.HelpName}} myminio/mybucket + + 2. Export replication configuration on bucket "mybucket" for alias "myminio" to '/data/replicate/config'. + {{.Prompt}} {{.HelpName}} myminio/mybucket > /data/replicate/config +`, +} + +// checkReplicateExportSyntax - validate all the passed arguments +func checkReplicateExportSyntax(ctx *cli.Context) { + if len(ctx.Args()) != 1 { + cli.ShowCommandHelpAndExit(ctx, "export", 1) // last argument is exit code + } +} + +type replicateExportMessage struct { + Op string `json:"op"` + Status string `json:"status"` + URL string `json:"url"` + ReplicationConfig replication.Config `json:"config"` +} + +func (r replicateExportMessage) JSON() string { + r.Status = "success" + jsonMessageBytes, e := json.MarshalIndent(r, "", " ") + fatalIf(probe.NewError(e), "Unable to marshal into JSON.") + return string(jsonMessageBytes) +} + +func (r replicateExportMessage) String() string { + if r.ReplicationConfig.Empty() { + return console.Colorize("ReplicateNMessage", "No replication configuration found for "+r.URL+".") + } + msgBytes, e := json.MarshalIndent(r.ReplicationConfig, "", " ") + fatalIf(probe.NewError(e), "Unable to marshal replication configuration") + return string(msgBytes) +} + +func mainReplicateExport(cliCtx *cli.Context) error { + ctx, cancelReplicateExport := context.WithCancel(globalContext) + defer cancelReplicateExport() + + console.SetColor("replicateExportMessage", color.New(color.FgGreen)) + console.SetColor("replicateExportFailure", color.New(color.FgRed)) + + checkReplicateExportSyntax(cliCtx) + + // Get the alias parameter from cli + args := cliCtx.Args() + aliasedURL := args.Get(0) + // Create a new Client + client, err := newClient(aliasedURL) + fatalIf(err, "Unable to initialize connection.") + rCfg, err := client.GetReplication(ctx) + fatalIf(err.Trace(args...), "Unable to get replication configuration") + printMsg(replicateExportMessage{ + Op: "export", + Status: "success", + URL: aliasedURL, + ReplicationConfig: rCfg, + }) + return nil +} diff --git a/cmd/replicate-import.go b/cmd/replicate-import.go new file mode 100644 index 00000000..280157de --- /dev/null +++ b/cmd/replicate-import.go @@ -0,0 +1,117 @@ +/* + * MinIO Client (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "context" + "os" + + "github.com/fatih/color" + "github.com/minio/cli" + json "github.com/minio/mc/pkg/colorjson" + "github.com/minio/mc/pkg/probe" + "github.com/minio/minio-go/v7/pkg/replication" + "github.com/minio/minio/pkg/console" +) + +var replicateImportCmd = cli.Command{ + Name: "import", + Usage: "import server side replication configuration in JSON format", + Action: mainReplicateImport, + Before: setGlobalsFromContext, + Flags: globalFlags, + CustomHelpTemplate: `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}} TARGET + +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}} +EXAMPLES: + 1. Set replication configuration from '/data/replication/config' on bucket "mybucket" for alias "myminio". + {{.Prompt}} {{.HelpName}} myminio/mybucket < '/data/replication/config' + + 2. Import replication configuration for bucket "mybucket" on alias "myminio" from STDIN. + {{.Prompt}} {{.HelpName}} myminio/mybucket +`, +} + +// checkReplicateImportSyntax - validate all the passed arguments +func checkReplicateImportSyntax(ctx *cli.Context) { + if len(ctx.Args()) != 1 { + cli.ShowCommandHelpAndExit(ctx, "import", 1) // last argument is exit code + } +} + +type replicateImportMessage struct { + Op string `json:"op"` + Status string `json:"status"` + URL string `json:"url"` + ReplicationConfig replication.Config `json:"config"` +} + +func (r replicateImportMessage) JSON() string { + r.Status = "success" + jsonMessageBytes, e := json.MarshalIndent(r, "", " ") + fatalIf(probe.NewError(e), "Unable to marshal into JSON.") + return string(jsonMessageBytes) +} + +func (r replicateImportMessage) String() string { + return console.Colorize("replicateImportMessage", "Replication configuration successfully set on `"+r.URL+"`.") +} + +// readReplicationConfig read from stdin, returns XML. +func readReplicationConfig() (*replication.Config, *probe.Error) { + // User is expected to enter the replication configuration in JSON format + var cfg = replication.Config{} + + // Consume json from STDIN + dec := json.NewDecoder(os.Stdin) + if e := dec.Decode(&cfg); e != nil { + return &cfg, probe.NewError(e) + } + + return &cfg, nil +} + +func mainReplicateImport(cliCtx *cli.Context) error { + ctx, cancelReplicateImport := context.WithCancel(globalContext) + defer cancelReplicateImport() + + console.SetColor("replicateImportMessage", color.New(color.FgGreen)) + checkReplicateImportSyntax(cliCtx) + + // Get the alias parameter from cli + args := cliCtx.Args() + aliasedURL := args.Get(0) + // Create a new Client + client, err := newClient(aliasedURL) + fatalIf(err, "Unable to initialize connection.") + rCfg, err := readReplicationConfig() + fatalIf(err.Trace(args...), "Unable to read replication configuration") + + fatalIf(client.SetReplication(ctx, rCfg, replication.Options{Op: replication.ImportOption}).Trace(aliasedURL), "Unable to set replication configuration") + printMsg(replicateImportMessage{ + Op: "import", + Status: "success", + URL: aliasedURL, + }) + return nil +} diff --git a/cmd/replicate-ls.go b/cmd/replicate-ls.go new file mode 100644 index 00000000..db2d47b0 --- /dev/null +++ b/cmd/replicate-ls.go @@ -0,0 +1,155 @@ +/* + * MinIO Client (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "context" + "errors" + "strconv" + "strings" + + "github.com/fatih/color" + "github.com/minio/cli" + json "github.com/minio/mc/pkg/colorjson" + "github.com/minio/mc/pkg/probe" + "github.com/minio/minio-go/v7/pkg/replication" + "github.com/minio/minio/pkg/console" +) + +var replicateListFlags = []cli.Flag{ + cli.StringFlag{ + Name: "status", + Usage: "show rules by status. Valid options are [enabled,disabled]", + }, +} + +var replicateListCmd = cli.Command{ + Name: "ls", + Usage: "list server side replication configuration rules", + Action: mainReplicateList, + Before: setGlobalsFromContext, + Flags: append(globalFlags, replicateListFlags...), + CustomHelpTemplate: `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}} TARGET + +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}} +EXAMPLES: + 1. List server side replication configuration rules on bucket "mybucket" for alias "myminio". + {{.Prompt}} {{.HelpName}} myminio/mybucket +`, +} + +// checkReplicateListSyntax - validate all the passed arguments +func checkReplicateListSyntax(ctx *cli.Context) { + if len(ctx.Args()) != 1 { + cli.ShowCommandHelpAndExit(ctx, "ls", 1) // last argument is exit code + } +} +func printReplicateListHeader() { + if globalJSON { + return + } + idFieldMaxLen := 20 + priorityFieldMaxLen := 8 + statusFieldMaxLen := 8 + prefixFieldMaxLen := 25 + tagsFieldMaxLen := 25 + scFieldMaxLen := 15 + destBucketFieldMaxLen := 20 + console.Println(console.Colorize("Headers", newPrettyTable(" | ", + Field{"ID", idFieldMaxLen}, + Field{"Priority", priorityFieldMaxLen}, + Field{"Status", statusFieldMaxLen}, + Field{"Prefix", prefixFieldMaxLen}, + Field{"Tags", tagsFieldMaxLen}, + Field{"DestBucket", destBucketFieldMaxLen}, + Field{"StorageClass", scFieldMaxLen}, + ).buildRow("ID", "Priority", "Status", "Prefix", "Tags", "DestBucket", "StorageClass"))) +} + +type replicateListMessage struct { + Op string `json:"op"` + Status string `json:"status"` + URL string `json:"url"` + Rule replication.Rule `json:"rule"` +} + +func (l replicateListMessage) JSON() string { + l.Status = "success" + jsonMessageBytes, e := json.MarshalIndent(l, "", " ") + fatalIf(probe.NewError(e), "Unable to marshal into JSON.") + return string(jsonMessageBytes) +} + +func (l replicateListMessage) String() string { + idFieldMaxLen := 20 + priorityFieldMaxLen := 8 + statusFieldMaxLen := 8 + prefixFieldMaxLen := 25 + tagsFieldMaxLen := 25 + scFieldMaxLen := 15 + destBucketFieldMaxLen := 20 + r := l.Rule + return console.Colorize("replicateListMessage", newPrettyTable(" | ", + Field{"ID", idFieldMaxLen}, + Field{"Priority", priorityFieldMaxLen}, + Field{"Status", statusFieldMaxLen}, + Field{"Prefix", prefixFieldMaxLen}, + Field{"Tags", tagsFieldMaxLen}, + Field{"DestBucket", destBucketFieldMaxLen}, + Field{"StorageClass", scFieldMaxLen}, + ).buildRow(r.ID, strconv.Itoa(r.Priority), string(r.Status), r.Filter.And.Prefix, r.Tags(), r.Destination.Bucket, r.Destination.StorageClass)) +} + +func mainReplicateList(cliCtx *cli.Context) error { + ctx, cancelReplicateList := context.WithCancel(globalContext) + defer cancelReplicateList() + + console.SetColor("Headers", color.New(color.Bold, color.FgHiGreen)) + + checkReplicateListSyntax(cliCtx) + + // Get the alias parameter from cli + args := cliCtx.Args() + aliasedURL := args.Get(0) + // Create a new Client + client, err := newClient(aliasedURL) + fatalIf(err, "Unable to initialize connection.") + rCfg, err := client.GetReplication(ctx) + fatalIf(err.Trace(args...), "Unable to get replication configuration") + + if rCfg.Empty() { + fatalIf(probe.NewError(errors.New("replication configuration not set")).Trace(aliasedURL), + "Unable to list replication configuration") + } + printReplicateListHeader() + statusFlag := cliCtx.String("status") + for _, rule := range rCfg.Rules { + if statusFlag == "" || strings.EqualFold(statusFlag, string(rule.Status)) { + printMsg(replicateListMessage{ + Rule: rule, + }) + } + } + + return nil +} diff --git a/cmd/replicate-main.go b/cmd/replicate-main.go new file mode 100644 index 00000000..978407b0 --- /dev/null +++ b/cmd/replicate-main.go @@ -0,0 +1,43 @@ +/* + * MinIO Client (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import "github.com/minio/cli" + +var replicateCmd = cli.Command{ + Name: "replicate", + Usage: "manage bucket server side replication", + HideHelpCommand: true, + Action: mainReplicate, + Before: setGlobalsFromContext, + Flags: globalFlags, + Subcommands: []cli.Command{ + replicateAddCmd, + replicateSetCmd, + replicateListCmd, + replicateExportCmd, + replicateImportCmd, + replicateRemoveCmd, + }, +} + +// mainReplicate is the handle for "mc replicate" command. +func mainReplicate(ctx *cli.Context) error { + cli.ShowCommandHelp(ctx, ctx.Args().First()) + return nil + // Sub-commands like "list", "clear", "add" have their own main. +} diff --git a/cmd/replicate-rm.go b/cmd/replicate-rm.go new file mode 100644 index 00000000..a6adae7c --- /dev/null +++ b/cmd/replicate-rm.go @@ -0,0 +1,154 @@ +/* + * MinIO Client (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "context" + "errors" + + "github.com/fatih/color" + "github.com/minio/cli" + json "github.com/minio/mc/pkg/colorjson" + "github.com/minio/mc/pkg/probe" + "github.com/minio/minio-go/v7/pkg/replication" + "github.com/minio/minio/pkg/console" +) + +var replicateRemoveFlags = []cli.Flag{ + cli.StringFlag{ + Name: "id", + Usage: "id for the rule, should be a unique value", + }, + cli.BoolFlag{ + Name: "force", + Usage: "force remove all the replication configuration rules on the bucket", + }, + cli.BoolFlag{ + Name: "all", + Usage: "remove all replication configuration rules of the bucket, force flag enforced", + }, +} + +var replicateRemoveCmd = cli.Command{ + Name: "rm", + Usage: "remove a server side replication configuration rule", + Action: mainReplicateRemove, + Before: setGlobalsFromContext, + Flags: append(globalFlags, replicateRemoveFlags...), + CustomHelpTemplate: `NAME: + {{.HelpName}} - {{.Usage}} + + USAGE: + {{.HelpName}} TARGET + +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}} +EXAMPLES: + 1. Remove replication configuration rule on bucket "mybucket" for alias "myminio" with rule id "bsib5mgt874bi56l0fmg". + {{.Prompt}} {{.HelpName}} --id "bsib5mgt874bi56l0fmg" myminio/mybucket + + 2. Remove all the replication configuration rules on bucket "mybucket" for alias "myminio". --force flag is required. + {{.Prompt}} {{.HelpName}} --all --force myminio/mybucket +`, +} + +// checkReplicateRemoveSyntax - validate all the passed arguments +func checkReplicateRemoveSyntax(ctx *cli.Context) { + if len(ctx.Args()) != 1 { + cli.ShowCommandHelpAndExit(ctx, "rm", 1) // last argument is exit code + } + rmAll := ctx.Bool("all") + rmForce := ctx.Bool("force") + rID := ctx.String("id") + + rmChk := (rmAll && rmForce) || (!rmAll && !rmForce) + if !rmChk { + fatalIf(errInvalidArgument(), + "It is mandatory to specify --all and --force flag together for mc "+ctx.Command.FullName()+".") + } + if rmAll && rmForce { + return + } + + if rID == "" { + fatalIf(errInvalidArgument().Trace(rID), "rule ID cannot be empty") + } +} + +type replicateRemoveMessage struct { + Op string `json:"op"` + Status string `json:"status"` + URL string `json:"url"` + ID string `json:"id"` +} + +func (l replicateRemoveMessage) JSON() string { + l.Status = "success" + jsonMessageBytes, e := json.MarshalIndent(l, "", " ") + fatalIf(probe.NewError(e), "Unable to marshal into JSON.") + return string(jsonMessageBytes) +} + +func (l replicateRemoveMessage) String() string { + if l.ID != "" { + return console.Colorize("replicateRemoveMessage", "Replication configuration rule with ID `"+l.ID+"`removed from "+l.URL+".") + } + return console.Colorize("replicateRemoveMessage", "Replication configuration removed from "+l.URL+" successfully.") +} + +func mainReplicateRemove(cliCtx *cli.Context) error { + ctx, cancelReplicateRemove := context.WithCancel(globalContext) + defer cancelReplicateRemove() + + console.SetColor("replicateRemoveMessage", color.New(color.FgGreen)) + + checkReplicateRemoveSyntax(cliCtx) + + // Get the alias parameter from cli + args := cliCtx.Args() + aliasedURL := args.Get(0) + // Create a new Client + client, err := newClient(aliasedURL) + fatalIf(err, "Unable to initialize connection.") + rcfg, err := client.GetReplication(ctx) + fatalIf(err.Trace(args...), "Unable to get replication configuration") + + if rcfg.Empty() { + fatalIf(probe.NewError(errors.New("replication configuration not set")).Trace(aliasedURL), + "Unable to remove replication configuration") + } + rmAll := cliCtx.Bool("all") + rmForce := cliCtx.Bool("force") + ruleID := cliCtx.String("id") + if rmAll && rmForce { + fatalIf(client.RemoveReplication(ctx), "Cannot remove replication configuration") + } else { + opts := replication.Options{ + ID: ruleID, + Op: replication.RemoveOption, + } + fatalIf(client.SetReplication(ctx, &rcfg, opts), "Could not remove replication rule") + } + printMsg(replicateRemoveMessage{ + Op: "rm", + Status: "success", + URL: aliasedURL, + ID: ruleID, + }) + return nil +} diff --git a/cmd/replicate-set.go b/cmd/replicate-set.go new file mode 100644 index 00000000..371b0ebf --- /dev/null +++ b/cmd/replicate-set.go @@ -0,0 +1,160 @@ +/* + * MinIO Client (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "context" + "strconv" + "strings" + + "github.com/fatih/color" + "github.com/minio/cli" + json "github.com/minio/mc/pkg/colorjson" + "github.com/minio/mc/pkg/probe" + "github.com/minio/minio-go/v7/pkg/replication" + "github.com/minio/minio/pkg/console" +) + +var replicateSetFlags = []cli.Flag{ + cli.StringFlag{ + Name: "id", + Usage: "id for the rule, should be a unique value", + }, + cli.StringFlag{ + Name: "tags", + Usage: "format '=&=&=', multiple values allowed for multiple key/value pairs", + }, + cli.StringFlag{ + Name: "storage-class", + Usage: "storage class for destination (STANDARD_IA,REDUCED_REDUNDANCY etc)", + }, + cli.StringFlag{ + Name: "state", + Usage: "change rule status. Valid values are [enable|disable]", + }, + cli.IntFlag{ + Name: "priority", + Usage: "priority of the rule, should be unique and is a required field", + }, +} + +var replicateSetCmd = cli.Command{ + Name: "set", + Usage: "modify an existing server side replication configuration rule", + Action: mainReplicateSet, + Before: setGlobalsFromContext, + Flags: append(globalFlags, replicateSetFlags...), + CustomHelpTemplate: `NAME: + {{.HelpName}} - {{.Usage}} + + USAGE: + {{.HelpName}} TARGET + + FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}} + EXAMPLES: + 1. Change priority of rule with rule ID "bsibgh8t874dnjst8hkg" on bucket "mybucket" for alias "myminio". + {{.Prompt}} {{.HelpName}} myminio/mybucket --id "bsibgh8t874dnjst8hkg" --priority 3 + + 2. Disable a replication configuration rule with rule ID "bsibgh8t874dnjst8hkg" on target myminio/bucket + {{.Prompt}} {{.HelpName}} myminio/mybucket --id "bsibgh8t874dnjst8hkg" --state disable + + 3. Set tags and storage class on a replication configuration with rule ID "kMYD.491" on target myminio/bucket/prefix. + {{.Prompt}} {{.HelpName}} myminio/mybucket --id "kMYD.491" --tags "key1=value1&key2=value2" \ + --storage-class "STANDARD" --priority 2 + + 4. Clear tags for replication configuration rule with ID "kMYD.491" on a target myminio/bucket. + {{.Prompt}} {{.HelpName}} myminio/mybucket --id "kMYD.491" --tags "" +`, +} + +// checkReplicateSetSyntax - validate all the passed arguments +func checkReplicateSetSyntax(ctx *cli.Context) { + if len(ctx.Args()) != 1 { + cli.ShowCommandHelpAndExit(ctx, "set", 1) // last argument is exit code + } +} + +type replicateSetMessage struct { + Op string `json:"op"` + Status string `json:"status"` + URL string `json:"url"` + ID string `json:"id"` +} + +func (l replicateSetMessage) JSON() string { + l.Status = "success" + jsonMessageBytes, e := json.MarshalIndent(l, "", " ") + fatalIf(probe.NewError(e), "Unable to marshal into JSON.") + return string(jsonMessageBytes) +} + +func (l replicateSetMessage) String() string { + if l.ID != "" { + return console.Colorize("replicateSetMessage", "Replication configuration rule with ID `"+l.ID+"` applied to "+l.URL+".") + } + return console.Colorize("replicateSetMessage", "Replication configuration rule applied to "+l.URL+" successfully.") +} + +func mainReplicateSet(cliCtx *cli.Context) error { + ctx, cancelReplicateSet := context.WithCancel(globalContext) + defer cancelReplicateSet() + + console.SetColor("replicateSetMessage", color.New(color.FgGreen)) + + checkReplicateSetSyntax(cliCtx) + + // Get the alias parameter from cli + args := cliCtx.Args() + aliasedURL := args.Get(0) + // Create a new Client + client, err := newClient(aliasedURL) + fatalIf(err, "Unable to initialize connection.") + rcfg, err := client.GetReplication(ctx) + fatalIf(err.Trace(args...), "Unable to get replication configuration") + + if !cliCtx.IsSet("id") { + fatalIf(errInvalidArgument(), "--id is a required flag") + } + var state string + if cliCtx.IsSet("state") { + state = strings.ToLower(cliCtx.String("state")) + if state != "enable" && state != "disable" { + fatalIf(err.Trace(args...), "--state can be either `enable` or `disable`") + } + } + opts := replication.Options{ + TagString: cliCtx.String("tags"), + Arn: cliCtx.String("arn"), + StorageClass: cliCtx.String("storage-class"), + RuleStatus: state, + ID: cliCtx.String("id"), + Op: replication.SetOption, + } + if cliCtx.IsSet("priority") { + opts.Priority = strconv.Itoa(cliCtx.Int("priority")) + } + + fatalIf(client.SetReplication(ctx, &rcfg, opts), "Could not modify replication rule") + printMsg(replicateSetMessage{ + Op: "set", + URL: aliasedURL, + ID: opts.ID, + }) + return nil +} diff --git a/docs/minio-admin-complete-guide.md b/docs/minio-admin-complete-guide.md index 07342a08..d77b3f1c 100644 --- a/docs/minio-admin-complete-guide.md +++ b/docs/minio-admin-complete-guide.md @@ -894,7 +894,7 @@ mc admin bucket quota myminio/mybucket --clear ``` -### Command `remote` - Manage bucket targets +### Command `remote` - manage bucket targets `remote` command manages remote bucket targets on MinIO server. ``` @@ -905,7 +905,7 @@ USAGE: mc admin bucket remote set TARGET http(s)://ACCESSKEY:SECRETKEY@TARGET_URL/TARGET_BUCKET [--path | --api] --type TARGET_BUCKET: - Also called as target bucket. + Also called as remote bucket. TARGET_URL: Also called as remote endpoint. diff --git a/docs/minio-client-complete-guide.md b/docs/minio-client-complete-guide.md index f5671ee3..ab031437 100644 --- a/docs/minio-client-complete-guide.md +++ b/docs/minio-client-complete-guide.md @@ -31,6 +31,7 @@ event manage object notifications watch listen for object notification events policy manage anonymous access to buckets and objects tag manage tags for bucket(s) and object(s) +replicate manage bucket server side replication admin manage MinIO servers update update mc to latest release ``` @@ -304,7 +305,7 @@ mc version RELEASE.2020-04-25T00-43-23Z | [**ls** - list buckets and objects](#ls) | [**tree** - list buckets and objects in a tree format](#tree) | [**mb** - make a bucket](#mb) | [**cat** - display object contents](#cat) | | [**cp** - copy objects](#cp) | [**rb** - remove a bucket](#rb) | [**pipe** - stream STDIN to an object](#pipe) | [**version** - manage bucket version](#version) | | [**share** - generate URL for temporary access to an object](#share) | [**rm** - remove objects](#rm) | [**find** - find files and objects](#find) | | -| [**diff** - list differences in object name, size, and date between two buckets](#diff) | [**mirror** - synchronize object(s) to a remote site](#mirror) | [**ilm** - manage bucket lifecycle policies](#ilm) | | +| [**diff** - list differences in object name, size, and date between two buckets](#diff) | [**mirror** - synchronize object(s) to a remote site](#mirror) | [**ilm** - manage bucket lifecycle policies](#ilm) | [**replicate** - manage bucket server side replication](#replicate) | | [**alias** - manage aliases](#alias) | [**policy** - set public policy on bucket or prefix](#policy) | [**event** - manage events on your buckets](#event) | | | [**update** - manage software updates](#update) | [**watch** - watch for events](#watch) | [**stat** - stat contents of objects and folders](#stat) | [**encrypt** - manage bucket encryption](#encrypt) | | [**head** - display first 'n' lines of an object](#head) | [**lock** - manage default bucket object lock configuration](#lock) | [**retention** - set retention for object(s)](#retention) | | @@ -1586,3 +1587,86 @@ Auto encryption has been set successfully for myminio/source mc encrypt clear myminio/mybucket Auto encryption configuration has been cleared successfully. ``` + + +### Command `replicate` +`replicate` manages bucket server side replication + +``` +NAME: + mc replicate - manage bucket server side replication + +USAGE: + mc replicate COMMAND [COMMAND FLAGS | -h] [ARGUMENTS...] + +COMMANDS: + add add a server side replication configuration rule + set modify an existing server side replication cofiguration rule + ls list server side replication configuration rules + export export server side replication configuration + import import server side replication configuration in JSON format + rm remove a server side replication configuration rule(s) + +FLAGS: + --help, -h show help +``` + +*Example: Add replication configuration rule on `mybucket` on alias `myminio`* + +``` +mc replicate add myminio/mybucket/prefix --tags "key1=value1&key2=value2" --storage-class "STANDARD" --arn 'arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket' --priority 1 +Replication configuration rule applied to myminio/mybucket/prefix. +``` + +*Example: Disable replication configuration rule with rule Id "bsibgh8t874dnjst8hkg" on bucket "mybucket" with prefix "prefix" for alias `myminio`* + +``` +mc replicate set myminio/mybucket/prefix --id "bsibgh8t874dnjst8hkg" --state disable +Replication configuration rule with ID `bsibgh8t874dnjst8hkg` applied to myminio/mybucket/prefix. +``` +*Example: Change priority of rule with rule ID "bsibgh8t874dnjst8hkg" on bucket "mybucket" for alias `myminio`.* + +``` +mc replicate set myminio/mybucket/prefix --id "bsibgh8t874dnjst8hkg" --priority 3 +Replication configuration rule with ID `bsibgh8t874dnjst8hkg` applied to myminio/mybucket/prefix. +``` + +*Example: Clear tags on rule ID "bsibgh8t874dnjst8hkg" for target myminio/bucket which has a replication configuration rule with prefix "prefix" + +``` +mc replicate set myminio/mybucket/prefix --id "bsibgh8t874dnjst8hkg" --tags "" +Replication configuration rule with ID `bsibgh8t874dnjst8hkg` applied to myminio/mybucket/prefix successfully. +``` + +*Example: List replication configuration rules set on `mybucket` on alias `myminio`* + +``` +mc replicate ls myminio/mybucket +``` + +*Example: Clear replication configuration for bucket `mybucket` on alias `myminio`* + +``` +mc replicate rm --all --force myminio/mybucket +Replication configuration has been removed successfully for myminio/mybucket +``` + +*Example: Remove replication configuration rule with id `bsibgh8t874dnjst8hkg` for bucket `mybucket` on alias `myminio`* + +``` +mc replicate rm --id "bsibgh8t874dnjst8hkg" myminio/mybucket/prefix +Replication configuration rule with id "bsibgh8t874dnjst8hkg" has been removed successfully for myminio/mybucket +``` + +*Example: Import replication configuration for bucket `mybucket` on alias `myminio` from `/data/replicate/config`* + +``` +mc replicate import myminio/mybucket < /data/replicate/config +Replication configuration successfully set on `myminio/mybucket`. +``` + +*Example: Export replication configuration for bucket `mybucket` on alias `myminio` to `/data/replicate/config`* + +``` +mc replicate export myminio/mybucket > /data/replicate/config +``` diff --git a/go.mod b/go.mod index f5978e53..7c63adde 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/mattn/go-isatty v0.0.8 github.com/minio/cli v1.22.0 github.com/minio/minio v0.0.0-20200731035804-2174a228351a - github.com/minio/minio-go/v7 v7.0.3 + github.com/minio/minio-go/v7 v7.0.5-0.20200807085956-d7db33ea7618 github.com/minio/sha256-simd v0.1.1 github.com/mitchellh/go-homedir v1.1.0 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect diff --git a/go.sum b/go.sum index 3975b5ae..469c0fee 100644 --- a/go.sum +++ b/go.sum @@ -279,10 +279,10 @@ github.com/minio/minio v0.0.0-20200731035804-2174a228351a h1:bM3ZSzv4N5oRKKIShsL github.com/minio/minio v0.0.0-20200731035804-2174a228351a/go.mod h1:1RkMjE0Ujyv+Cb5vjNIrO85b3XFlfMMEFmma4hOEKDk= github.com/minio/minio-go/v7 v7.0.1 h1:sL2y4uuNUEi7AjvWjoGyDFQKFX2zA0DU2tGM9m3s5f8= github.com/minio/minio-go/v7 v7.0.1/go.mod h1:dJ80Mv2HeGkYLH1sqS/ksz07ON6csH3S6JUMSQ2zAns= -github.com/minio/minio-go/v7 v7.0.2-0.20200722162308-e0105ca08252 h1:V2JkMDoSmEIhRcMJwX3qeJVOzy1B5bHpHbZaQu77vbs= -github.com/minio/minio-go/v7 v7.0.2-0.20200722162308-e0105ca08252/go.mod h1:dJ80Mv2HeGkYLH1sqS/ksz07ON6csH3S6JUMSQ2zAns= -github.com/minio/minio-go/v7 v7.0.3 h1:a2VHaXDlKBcB3J5XJhKVfWBRi1+ZmMWFXABQ8TLlWbA= -github.com/minio/minio-go/v7 v7.0.3/go.mod h1:TA0CQCjJZHM5SJj9IjqR0NmpmQJ6bCbXifAJ3mUU6Hw= +github.com/minio/minio-go/v7 v7.0.4 h1:M9glnGclD87VfttesWzURw7SHqq1XDIYGrfTykBTI50= +github.com/minio/minio-go/v7 v7.0.4/go.mod h1:CSt2ETZNs+bIIhWTse0mcZKZWMGrFU7Er7RR0TmkDYk= +github.com/minio/minio-go/v7 v7.0.5-0.20200807085956-d7db33ea7618 h1:8iTb0TFs6kDGAUnhI/s2QCZOYcSTtYmY9dF+Cbc0WJo= +github.com/minio/minio-go/v7 v7.0.5-0.20200807085956-d7db33ea7618/go.mod h1:CSt2ETZNs+bIIhWTse0mcZKZWMGrFU7Er7RR0TmkDYk= github.com/minio/selfupdate v0.3.0/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= @@ -533,8 +533,6 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20u golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666 h1:gVCS+QOncANNPlmlO1AhlU3oxs4V9z+gTtPwIk3p2N8= -golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200806125547-5acd03effb82 h1:6cBnXxYO+CiRVrChvCosSv7magqTPbyAgz1M8iOv5wM= golang.org/x/sys v0.0.0-20200806125547-5acd03effb82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=