fix: enhance support for special access/secretkeys in remote targets (#3725)

it latest release and master `mc admin bucket remote add` cannot
parse accesskeys/secretkeys with special characters such as `@`,
this is due to an incorrect implementation.

use the technique used in parsing MC_HOST_alias here to allow
for more complex use-cases.

cmd/admin-bucket-remote-add.go

@@ -20,8 +20,7 @@ package cmd
 import (
 	"fmt"
 	"net/url"
-	"regexp"
+	"path"
-	"strings"
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
@@ -30,6 +29,7 @@ import (
 	json "github.com/minio/colorjson"
 	"github.com/minio/madmin-go"
 	"github.com/minio/mc/pkg/probe"
+	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio/pkg/console"
 )
 
@@ -168,7 +168,31 @@ func (r RemoteMessage) JSON() string {
 	return string(jsonMessageBytes)
 }
 
-var targetKeys = regexp.MustCompile("^(https?://)(.*?):(.*?)@(.*?)/(.*?)$")
+func extractCredentialURL(argURL string) (accessKey, secretKey string, u *url.URL) {
+	var parsedURL string
+	if hostKeyTokens.MatchString(argURL) {
+		fatalIf(errInvalidArgument().Trace(argURL), "temporary tokens are not allowed for remote targets")
+	}
+	if hostKeys.MatchString(argURL) {
+		parts := hostKeys.FindStringSubmatch(argURL)
+		if len(parts) != 5 {
+			fatalIf(errInvalidArgument().Trace(argURL), "Unsupported remote target format, please check --help")
+		}
+		accessKey = parts[2]
+		secretKey = parts[3]
+		parsedURL = fmt.Sprintf("%s%s", parts[1], parts[4])
+	}
+	var e error
+	if parsedURL == "" {
+		fatalIf(errInvalidArgument().Trace(argURL), "No valid credentials were detected")
+	}
+	u, e = url.Parse(parsedURL)
+	if e != nil {
+		fatalIf(errInvalidArgument().Trace(parsedURL), "Unsupported URL format %v", e)
+	}
+
+	return accessKey, secretKey, u
+}
 
 // fetchRemoteTarget - returns the dest bucket, dest endpoint, access and secret key
 func fetchRemoteTarget(cli *cli.Context) (sourceBucket string, bktTarget *madmin.BucketTarget) {
@@ -178,24 +202,20 @@ func fetchRemoteTarget(cli *cli.Context) (sourceBucket string, bktTarget *madmin
 		fatalIf(probe.NewError(fmt.Errorf("Missing Remote target configuration")), "Unable to parse remote target")
 	}
 	_, sourceBucket = url2Alias(args[0])
-	TargetURL := args[1]
+	p := cli.String("path")
-	path := cli.String("path")
+	if !isValidPath(p) {
-	if !isValidPath(path) {
+		fatalIf(errInvalidArgument().Trace(p),
-		fatalIf(errInvalidArgument().Trace(path),
 			"Unrecognized bucket path style. Valid options are `[on,off, auto]`.")
 	}
-	parts := targetKeys.FindStringSubmatch(TargetURL)
+
-	if len(parts) != 6 {
+	tgtURL := args[1]
-		fatalIf(probe.NewError(fmt.Errorf("invalid url format")), "Malformed Remote target URL")
+	accessKey, secretKey, u := extractCredentialURL(tgtURL)
+	var tgtBucket string
+	if u.Path != "" {
+		tgtBucket = path.Clean(u.Path[1:])
 	}
-	accessKey := parts[2]
+	if e := s3utils.CheckValidBucketName(tgtBucket); e != nil {
-	secretKey := parts[3]
+		fatalIf(probe.NewError(e).Trace(tgtURL), "Invalid target bucket specified")
-	parsedURL := fmt.Sprintf("%s%s", parts[1], parts[4])
-	TargetBucket := strings.TrimSuffix(parts[5], slashSeperator)
-	TargetBucket = strings.TrimPrefix(TargetBucket, slashSeperator)
-	u, cerr := url.Parse(parsedURL)
-	if cerr != nil {
-		fatalIf(probe.NewError(cerr), "Malformed Remote target URL")
 	}
 
 	serviceType := cli.String("service")
@@ -213,11 +233,11 @@ func fetchRemoteTarget(cli *cli.Context) (sourceBucket string, bktTarget *madmin
 	console.SetColor(cred, color.New(color.FgYellow, color.Italic))
 	creds := &madmin.Credentials{AccessKey: accessKey, SecretKey: secretKey}
 	bktTarget = &madmin.BucketTarget{
-		TargetBucket:        TargetBucket,
+		TargetBucket:        tgtBucket,
 		Secure:              u.Scheme == "https",
 		Credentials:         creds,
 		Endpoint:            u.Host,
-		Path:                path,
+		Path:                p,
 		API:                 "s3v4",
 		Type:                madmin.ServiceType(serviceType),
 		Region:              cli.String("region"),

cmd/admin-bucket-remote-edit.go

@@ -18,15 +18,14 @@
 package cmd
 
 import (
-	"fmt"
+	"path"
-	"net/url"
 	"strconv"
-	"strings"
 
 	"github.com/fatih/color"
 	"github.com/minio/cli"
 	"github.com/minio/madmin-go"
 	"github.com/minio/mc/pkg/probe"
+	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio/pkg/console"
 )
 
@@ -88,20 +87,17 @@ func checkAdminBucketRemoteEditSyntax(ctx *cli.Context) {
 func fetchRemoteEditTarget(cli *cli.Context) (bktTarget *madmin.BucketTarget) {
 	args := cli.Args()
 	_, sourceBucket := url2Alias(args[0])
-	TargetURL := args[1]
+
-	parts := targetKeys.FindStringSubmatch(TargetURL)
+	tgtURL := args[1]
-	if len(parts) != 6 {
+	accessKey, secretKey, u := extractCredentialURL(tgtURL)
-		fatalIf(probe.NewError(fmt.Errorf("invalid url format")), "Malformed Remote target URL")
+	var tgtBucket string
+	if u.Path != "" {
+		tgtBucket = path.Clean(u.Path[1:])
 	}
-	accessKey := parts[2]
+	if e := s3utils.CheckValidBucketName(tgtBucket); e != nil {
-	secretKey := parts[3]
+		fatalIf(probe.NewError(e).Trace(tgtURL), "Invalid target bucket specified")
-	parsedURL := fmt.Sprintf("%s%s", parts[1], parts[4])
-	TargetBucket := strings.TrimSuffix(parts[5], slashSeperator)
-	TargetBucket = strings.TrimPrefix(TargetBucket, slashSeperator)
-	u, cerr := url.Parse(parsedURL)
-	if cerr != nil {
-		fatalIf(probe.NewError(cerr), "Malformed Remote target URL")
 	}
+
 	secure := u.Scheme == "https"
 	host := u.Host
 	if u.Port() == "" {
@@ -115,7 +111,7 @@ func fetchRemoteEditTarget(cli *cli.Context) (bktTarget *madmin.BucketTarget) {
 	creds := &madmin.Credentials{AccessKey: accessKey, SecretKey: secretKey}
 	bktTarget = &madmin.BucketTarget{
 		SourceBucket: sourceBucket,
-		TargetBucket: TargetBucket,
+		TargetBucket: tgtBucket,
 		Secure:       secure,
 		Credentials:  creds,
 		Endpoint:     host,

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/mattn/go-isatty v0.0.12
 	github.com/minio/cli v1.22.0
 	github.com/minio/colorjson v1.0.0
-	github.com/minio/filepath v1.0.0 // indirect
+	github.com/minio/filepath v1.0.0
 	github.com/minio/madmin-go v1.0.2
 	github.com/minio/minio v0.0.0-20210422165109-3455f786faf0
 	github.com/minio/minio-go/v7 v7.0.11-0.20210517200026-f0518ca447d6

go.sum

@@ -331,10 +331,6 @@ github.com/minio/md5-simd v1.1.1/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77Z
 github.com/minio/minio v0.0.0-20210422165109-3455f786faf0 h1:zKmm7kp4HfzMT7ImqFuNwBBo4Ne5ExntL92cggACA5M=
 github.com/minio/minio v0.0.0-20210422165109-3455f786faf0/go.mod h1:nFVEfjWoCj2KxWymJnQuVPolrE3/gvFCYm0wZkCIdXw=
 github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw=
-github.com/minio/minio-go/v7 v7.0.11-0.20210407221404-ba867dba7ee1 h1:x0jnjGGnb0PaewBIH9dbTaLJUNVKYoQOjjdqwz9PwdQ=
-github.com/minio/minio-go/v7 v7.0.11-0.20210407221404-ba867dba7ee1/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw=
-github.com/minio/minio-go/v7 v7.0.11-0.20210511181606-0263c8eee163 h1:kRHruZzRnERrnkv6EQvrQMIPdwW5rU77ekqlb7wZTjw=
-github.com/minio/minio-go/v7 v7.0.11-0.20210511181606-0263c8eee163/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo=
 github.com/minio/minio-go/v7 v7.0.11-0.20210517200026-f0518ca447d6 h1:GVR+UTvfe2r2YTYHWrA/yRF5nouMjJh3kwxNTZ8npso=
 github.com/minio/minio-go/v7 v7.0.11-0.20210517200026-f0518ca447d6/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo=
 github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM=
@@ -777,6 +773,7 @@ honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 maze.io/x/duration v0.0.0-20160924141736-faac084b6075 h1:4zVed9rL46683x3koxOYLzh8FlLFjnRrzTo2uvgA5D4=
 maze.io/x/duration v0.0.0-20160924141736-faac084b6075/go.mod h1:1kfR2ph3CIvtfIQ8D8JhmAgePmnAUnR+AWYWUBo+l08=